2024-11-15 11:51:05,970 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 11:51:05,980 main DEBUG Took 0.008654 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 11:51:05,981 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 11:51:05,981 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 11:51:05,982 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 11:51:05,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:05,989 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 11:51:05,999 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,002 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,003 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,004 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,005 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,006 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,007 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,008 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,009 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 11:51:06,010 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,010 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 11:51:06,011 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 11:51:06,013 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 11:51:06,014 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 11:51:06,015 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 11:51:06,016 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 11:51:06,016 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 11:51:06,023 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 11:51:06,026 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 11:51:06,027 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 11:51:06,028 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 11:51:06,028 main DEBUG createAppenders(={Console}) 2024-11-15 11:51:06,029 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 11:51:06,029 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 11:51:06,029 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 11:51:06,030 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 11:51:06,030 main DEBUG OutputStream closed 2024-11-15 11:51:06,030 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 11:51:06,030 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 11:51:06,031 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 11:51:06,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 11:51:06,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 11:51:06,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 11:51:06,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 11:51:06,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 11:51:06,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 11:51:06,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 11:51:06,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 11:51:06,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 11:51:06,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 11:51:06,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 11:51:06,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 11:51:06,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 11:51:06,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 11:51:06,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 11:51:06,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 11:51:06,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 11:51:06,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 11:51:06,106 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 11:51:06,106 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 11:51:06,107 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 11:51:06,107 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T11:51:06,331 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646 2024-11-15 11:51:06,334 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 11:51:06,334 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T11:51:06,341 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-15T11:51:06,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T11:51:06,377 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0, deleteOnExit=true 2024-11-15T11:51:06,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T11:51:06,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/test.cache.data in system properties and HBase conf 2024-11-15T11:51:06,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T11:51:06,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.log.dir in system properties and HBase conf 2024-11-15T11:51:06,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T11:51:06,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T11:51:06,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T11:51:06,468 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T11:51:06,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T11:51:06,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T11:51:06,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T11:51:06,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T11:51:06,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T11:51:06,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T11:51:06,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T11:51:06,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T11:51:06,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T11:51:06,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T11:51:06,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/nfs.dump.dir in system properties and HBase conf 2024-11-15T11:51:06,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/java.io.tmpdir in system properties and HBase conf 2024-11-15T11:51:06,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T11:51:06,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T11:51:06,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T11:51:07,585 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T11:51:07,651 INFO [Time-limited test {}] log.Log(170): Logging initialized @2334ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T11:51:07,714 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T11:51:07,769 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T11:51:07,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T11:51:07,791 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T11:51:07,793 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T11:51:07,806 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T11:51:07,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.log.dir/,AVAILABLE} 2024-11-15T11:51:07,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T11:51:07,981 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/java.io.tmpdir/jetty-localhost-39321-hadoop-hdfs-3_4_1-tests_jar-_-any-16391649151047106199/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T11:51:07,986 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39321} 2024-11-15T11:51:07,987 INFO [Time-limited test {}] server.Server(415): Started @2669ms 2024-11-15T11:51:08,495 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T11:51:08,504 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T11:51:08,505 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T11:51:08,505 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T11:51:08,506 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T11:51:08,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.log.dir/,AVAILABLE} 2024-11-15T11:51:08,507 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T11:51:08,619 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/java.io.tmpdir/jetty-localhost-34493-hadoop-hdfs-3_4_1-tests_jar-_-any-10511341898087599525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T11:51:08,619 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34493} 2024-11-15T11:51:08,620 INFO [Time-limited test {}] server.Server(415): Started @3302ms 2024-11-15T11:51:08,675 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T11:51:09,506 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data2/current/BP-588673048-172.17.0.2-1731671467135/current, will proceed with Du for space computation calculation, 2024-11-15T11:51:09,506 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data1/current/BP-588673048-172.17.0.2-1731671467135/current, will proceed with Du for space computation calculation, 2024-11-15T11:51:09,536 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T11:51:09,576 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646 2024-11-15T11:51:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1631f4d5cbc0386f with lease ID 0xb863b89ef42d5e79: Processing first storage report for DS-ce56c3b3-ec7f-433e-a304-720eb8a28922 from datanode DatanodeRegistration(127.0.0.1:42045, datanodeUuid=efe92078-cba5-4929-bf50-7576633a7deb, infoPort=42961, infoSecurePort=0, ipcPort=41181, storageInfo=lv=-57;cid=testClusterID;nsid=1234182401;c=1731671467135) 2024-11-15T11:51:09,580 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1631f4d5cbc0386f with lease ID 0xb863b89ef42d5e79: from storage DS-ce56c3b3-ec7f-433e-a304-720eb8a28922 node DatanodeRegistration(127.0.0.1:42045, datanodeUuid=efe92078-cba5-4929-bf50-7576633a7deb, infoPort=42961, infoSecurePort=0, ipcPort=41181, storageInfo=lv=-57;cid=testClusterID;nsid=1234182401;c=1731671467135), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T11:51:09,580 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1631f4d5cbc0386f with lease ID 0xb863b89ef42d5e79: Processing first storage report for DS-3466cd44-1498-456f-90f3-3299992d511f from datanode DatanodeRegistration(127.0.0.1:42045, datanodeUuid=efe92078-cba5-4929-bf50-7576633a7deb, infoPort=42961, infoSecurePort=0, ipcPort=41181, storageInfo=lv=-57;cid=testClusterID;nsid=1234182401;c=1731671467135) 2024-11-15T11:51:09,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1631f4d5cbc0386f with lease ID 0xb863b89ef42d5e79: from storage DS-3466cd44-1498-456f-90f3-3299992d511f node DatanodeRegistration(127.0.0.1:42045, datanodeUuid=efe92078-cba5-4929-bf50-7576633a7deb, infoPort=42961, infoSecurePort=0, ipcPort=41181, storageInfo=lv=-57;cid=testClusterID;nsid=1234182401;c=1731671467135), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T11:51:09,665 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/zookeeper_0, clientPort=54275, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T11:51:09,674 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54275 2024-11-15T11:51:09,684 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:09,687 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:09,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741825_1001 (size=7) 2024-11-15T11:51:10,313 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 with version=8 2024-11-15T11:51:10,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/hbase-staging 2024-11-15T11:51:10,398 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T11:51:10,634 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7adf9b3d9d04:0 server-side Connection retries=45 2024-11-15T11:51:10,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:10,643 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:10,648 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T11:51:10,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:10,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T11:51:10,776 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T11:51:10,841 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T11:51:10,850 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T11:51:10,854 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T11:51:10,879 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 86156 (auto-detected) 2024-11-15T11:51:10,880 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T11:51:10,898 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36957 2024-11-15T11:51:10,921 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36957 connecting to ZooKeeper ensemble=127.0.0.1:54275 2024-11-15T11:51:11,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369570x0, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T11:51:11,021 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36957-0x1013f9f41130000 connected 2024-11-15T11:51:11,098 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:11,100 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:11,108 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T11:51:11,112 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186, hbase.cluster.distributed=false 2024-11-15T11:51:11,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T11:51:11,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36957 2024-11-15T11:51:11,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36957 2024-11-15T11:51:11,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36957 2024-11-15T11:51:11,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36957 2024-11-15T11:51:11,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36957 2024-11-15T11:51:11,232 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7adf9b3d9d04:0 server-side Connection retries=45 2024-11-15T11:51:11,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:11,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:11,234 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T11:51:11,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T11:51:11,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T11:51:11,237 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T11:51:11,240 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T11:51:11,241 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44479 2024-11-15T11:51:11,243 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44479 connecting to ZooKeeper ensemble=127.0.0.1:54275 2024-11-15T11:51:11,244 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:11,247 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:11,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444790x0, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T11:51:11,280 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44479-0x1013f9f41130001 connected 2024-11-15T11:51:11,280 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T11:51:11,285 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T11:51:11,294 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T11:51:11,296 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T11:51:11,301 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T11:51:11,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44479 2024-11-15T11:51:11,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44479 2024-11-15T11:51:11,302 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44479 2024-11-15T11:51:11,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44479 2024-11-15T11:51:11,303 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44479 2024-11-15T11:51:11,319 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7adf9b3d9d04:36957 2024-11-15T11:51:11,320 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:11,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T11:51:11,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T11:51:11,348 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:11,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T11:51:11,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:11,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:11,380 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T11:51:11,382 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7adf9b3d9d04,36957,1731671470483 from backup master directory 2024-11-15T11:51:11,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T11:51:11,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:11,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T11:51:11,395 WARN [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T11:51:11,396 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:11,398 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T11:51:11,400 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T11:51:11,455 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/hbase.id] with ID: 52397efa-b1b0-4774-8091-97e7ccb77a0b 2024-11-15T11:51:11,455 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/.tmp/hbase.id 2024-11-15T11:51:11,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741826_1002 (size=42) 2024-11-15T11:51:11,869 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/.tmp/hbase.id]:[hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/hbase.id] 2024-11-15T11:51:11,926 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T11:51:11,931 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T11:51:11,951 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-15T11:51:12,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:12,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741827_1003 (size=196) 2024-11-15T11:51:12,460 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:51:12,463 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T11:51:12,477 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:12,481 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T11:51:12,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741828_1004 (size=1189) 2024-11-15T11:51:12,932 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store 2024-11-15T11:51:12,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741829_1005 (size=34) 2024-11-15T11:51:13,358 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T11:51:13,362 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:13,364 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T11:51:13,364 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:51:13,364 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:51:13,367 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-15T11:51:13,367 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:51:13,367 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:51:13,368 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731671473364Disabling compacts and flushes for region at 1731671473364Disabling writes for close at 1731671473367 (+3 ms)Writing region close event to WAL at 1731671473367Closed at 1731671473367 2024-11-15T11:51:13,371 WARN [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/.initializing 2024-11-15T11:51:13,371 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/WALs/7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:13,378 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T11:51:13,391 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7adf9b3d9d04%2C36957%2C1731671470483, suffix=, logDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/WALs/7adf9b3d9d04,36957,1731671470483, archiveDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/oldWALs, maxLogs=10 2024-11-15T11:51:13,411 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/WALs/7adf9b3d9d04,36957,1731671470483/7adf9b3d9d04%2C36957%2C1731671470483.1731671473395, exclude list is [], retry=0 2024-11-15T11:51:13,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42045,DS-ce56c3b3-ec7f-433e-a304-720eb8a28922,DISK] 2024-11-15T11:51:13,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T11:51:13,458 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/WALs/7adf9b3d9d04,36957,1731671470483/7adf9b3d9d04%2C36957%2C1731671470483.1731671473395 2024-11-15T11:51:13,459 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42961:42961)] 2024-11-15T11:51:13,460 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:51:13,460 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:13,463 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,464 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,499 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T11:51:13,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:13,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:13,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,531 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T11:51:13,531 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:13,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:13,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,535 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T11:51:13,535 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:13,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:13,536 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T11:51:13,539 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:13,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:13,540 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,543 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,544 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,550 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,550 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,555 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T11:51:13,559 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T11:51:13,564 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:51:13,565 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71326424, jitterRate=0.06284654140472412}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T11:51:13,573 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731671473475Initializing all the Stores at 1731671473477 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671473478 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671473479 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671473479Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671473479Cleaning up temporary data from old regions at 1731671473551 (+72 ms)Region opened successfully at 1731671473572 (+21 ms) 2024-11-15T11:51:13,574 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T11:51:13,609 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5ce06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7adf9b3d9d04/172.17.0.2:0 2024-11-15T11:51:13,638 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T11:51:13,647 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T11:51:13,647 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T11:51:13,649 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T11:51:13,651 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T11:51:13,655 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T11:51:13,655 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T11:51:13,678 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T11:51:13,686 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T11:51:13,745 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T11:51:13,750 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T11:51:13,753 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T11:51:13,762 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T11:51:13,764 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T11:51:13,769 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T11:51:13,778 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T11:51:13,780 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T11:51:13,787 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T11:51:13,806 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T11:51:13,811 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T11:51:13,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T11:51:13,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T11:51:13,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,823 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7adf9b3d9d04,36957,1731671470483, sessionid=0x1013f9f41130000, setting cluster-up flag (Was=false) 2024-11-15T11:51:13,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,879 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T11:51:13,882 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:13,928 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T11:51:13,930 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:13,937 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T11:51:14,004 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T11:51:14,010 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(746): ClusterId : 52397efa-b1b0-4774-8091-97e7ccb77a0b 2024-11-15T11:51:14,012 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T11:51:14,012 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T11:51:14,018 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T11:51:14,023 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7adf9b3d9d04,36957,1731671470483 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T11:51:14,030 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T11:51:14,030 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T11:51:14,030 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7adf9b3d9d04:0, corePoolSize=5, maxPoolSize=5 2024-11-15T11:51:14,030 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7adf9b3d9d04:0, corePoolSize=5, maxPoolSize=5 2024-11-15T11:51:14,030 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7adf9b3d9d04:0, corePoolSize=5, maxPoolSize=5 2024-11-15T11:51:14,031 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7adf9b3d9d04:0, corePoolSize=5, maxPoolSize=5 2024-11-15T11:51:14,031 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7adf9b3d9d04:0, corePoolSize=10, maxPoolSize=10 2024-11-15T11:51:14,031 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,031 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7adf9b3d9d04:0, corePoolSize=2, maxPoolSize=2 2024-11-15T11:51:14,031 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,033 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731671504033 2024-11-15T11:51:14,035 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T11:51:14,036 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T11:51:14,036 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T11:51:14,037 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T11:51:14,038 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T11:51:14,039 DEBUG [RS:0;7adf9b3d9d04:44479 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3bacfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7adf9b3d9d04/172.17.0.2:0 2024-11-15T11:51:14,039 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T11:51:14,040 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T11:51:14,040 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T11:51:14,040 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T11:51:14,042 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:14,041 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,043 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T11:51:14,045 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T11:51:14,047 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T11:51:14,047 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T11:51:14,054 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T11:51:14,055 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T11:51:14,056 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7adf9b3d9d04:44479 2024-11-15T11:51:14,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741831_1007 (size=1321) 2024-11-15T11:51:14,057 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.large.0-1731671474056,5,FailOnTimeoutGroup] 2024-11-15T11:51:14,058 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.small.0-1731671474057,5,FailOnTimeoutGroup] 2024-11-15T11:51:14,058 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,058 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T11:51:14,059 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T11:51:14,059 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T11:51:14,059 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T11:51:14,059 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,060 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,062 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(2659): reportForDuty to master=7adf9b3d9d04,36957,1731671470483 with port=44479, startcode=1731671471201 2024-11-15T11:51:14,072 DEBUG [RS:0;7adf9b3d9d04:44479 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T11:51:14,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55015, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T11:51:14,114 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36957 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,117 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36957 {}] master.ServerManager(517): Registering regionserver=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,132 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:51:14,132 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46663 2024-11-15T11:51:14,133 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T11:51:14,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T11:51:14,162 DEBUG [RS:0;7adf9b3d9d04:44479 {}] zookeeper.ZKUtil(111): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,163 WARN [RS:0;7adf9b3d9d04:44479 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T11:51:14,163 INFO [RS:0;7adf9b3d9d04:44479 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T11:51:14,163 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,165 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7adf9b3d9d04,44479,1731671471201] 2024-11-15T11:51:14,191 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T11:51:14,206 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T11:51:14,211 INFO [RS:0;7adf9b3d9d04:44479 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T11:51:14,211 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,212 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T11:51:14,219 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T11:51:14,221 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,221 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,221 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,221 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,222 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,222 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,222 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7adf9b3d9d04:0, corePoolSize=2, maxPoolSize=2 2024-11-15T11:51:14,222 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,222 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,223 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,223 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,223 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,223 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7adf9b3d9d04:0, corePoolSize=1, maxPoolSize=1 2024-11-15T11:51:14,223 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7adf9b3d9d04:0, corePoolSize=3, maxPoolSize=3 2024-11-15T11:51:14,224 DEBUG [RS:0;7adf9b3d9d04:44479 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0, corePoolSize=3, maxPoolSize=3 2024-11-15T11:51:14,225 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,225 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,225 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,225 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,225 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,226 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,44479,1731671471201-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T11:51:14,244 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T11:51:14,245 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,44479,1731671471201-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,245 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,246 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.Replication(171): 7adf9b3d9d04,44479,1731671471201 started 2024-11-15T11:51:14,268 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:14,268 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1482): Serving as 7adf9b3d9d04,44479,1731671471201, RpcServer on 7adf9b3d9d04/172.17.0.2:44479, sessionid=0x1013f9f41130001 2024-11-15T11:51:14,269 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T11:51:14,269 DEBUG [RS:0;7adf9b3d9d04:44479 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,270 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7adf9b3d9d04,44479,1731671471201' 2024-11-15T11:51:14,270 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T11:51:14,271 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T11:51:14,272 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T11:51:14,272 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T11:51:14,272 DEBUG [RS:0;7adf9b3d9d04:44479 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:14,272 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7adf9b3d9d04,44479,1731671471201' 2024-11-15T11:51:14,272 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T11:51:14,273 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T11:51:14,273 DEBUG [RS:0;7adf9b3d9d04:44479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T11:51:14,274 INFO [RS:0;7adf9b3d9d04:44479 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T11:51:14,274 INFO [RS:0;7adf9b3d9d04:44479 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T11:51:14,384 INFO [RS:0;7adf9b3d9d04:44479 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T11:51:14,388 INFO [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7adf9b3d9d04%2C44479%2C1731671471201, suffix=, logDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201, archiveDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/oldWALs, maxLogs=32 2024-11-15T11:51:14,403 DEBUG [RS:0;7adf9b3d9d04:44479 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201/7adf9b3d9d04%2C44479%2C1731671471201.1731671474390, exclude list is [], retry=0 2024-11-15T11:51:14,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42045,DS-ce56c3b3-ec7f-433e-a304-720eb8a28922,DISK] 2024-11-15T11:51:14,410 INFO [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201/7adf9b3d9d04%2C44479%2C1731671471201.1731671474390 2024-11-15T11:51:14,411 DEBUG [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42961:42961)] 2024-11-15T11:51:14,461 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T11:51:14,462 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:51:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741833_1009 (size=32) 2024-11-15T11:51:14,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:14,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T11:51:14,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T11:51:14,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:14,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:14,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T11:51:14,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T11:51:14,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:14,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:14,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T11:51:14,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T11:51:14,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:14,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:14,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T11:51:14,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T11:51:14,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:14,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:14,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T11:51:14,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740 2024-11-15T11:51:14,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740 2024-11-15T11:51:14,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T11:51:14,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T11:51:14,905 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:51:14,907 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T11:51:14,911 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:51:14,912 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68607526, jitterRate=0.022331804037094116}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:51:14,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731671474879Initializing all the Stores at 1731671474881 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671474881Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671474882 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671474882Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671474882Cleaning up temporary data from old regions at 1731671474904 (+22 ms)Region opened successfully at 1731671474914 (+10 ms) 2024-11-15T11:51:14,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T11:51:14,914 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T11:51:14,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T11:51:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T11:51:14,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T11:51:14,916 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T11:51:14,916 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731671474914Disabling compacts and flushes for region at 1731671474914Disabling writes for close at 1731671474915 (+1 ms)Writing region close event to WAL at 1731671474916 (+1 ms)Closed at 1731671474916 2024-11-15T11:51:14,919 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T11:51:14,920 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T11:51:14,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T11:51:14,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T11:51:14,935 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T11:51:15,088 DEBUG [7adf9b3d9d04:36957 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T11:51:15,098 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:15,104 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7adf9b3d9d04,44479,1731671471201, state=OPENING 2024-11-15T11:51:15,127 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T11:51:15,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:15,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:51:15,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T11:51:15,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T11:51:15,139 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T11:51:15,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:15,314 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T11:51:15,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T11:51:15,329 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T11:51:15,330 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T11:51:15,330 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T11:51:15,334 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7adf9b3d9d04%2C44479%2C1731671471201.meta, suffix=.meta, logDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201, archiveDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/oldWALs, maxLogs=32 2024-11-15T11:51:15,349 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201/7adf9b3d9d04%2C44479%2C1731671471201.meta.1731671475336.meta, exclude list is [], retry=0 2024-11-15T11:51:15,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42045,DS-ce56c3b3-ec7f-433e-a304-720eb8a28922,DISK] 2024-11-15T11:51:15,356 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/WALs/7adf9b3d9d04,44479,1731671471201/7adf9b3d9d04%2C44479%2C1731671471201.meta.1731671475336.meta 2024-11-15T11:51:15,357 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42961:42961)] 2024-11-15T11:51:15,357 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:51:15,359 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T11:51:15,361 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T11:51:15,366 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T11:51:15,371 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T11:51:15,372 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:15,372 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T11:51:15,372 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T11:51:15,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T11:51:15,377 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T11:51:15,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:15,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:15,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T11:51:15,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T11:51:15,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:15,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:15,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T11:51:15,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T11:51:15,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:15,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:15,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T11:51:15,385 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T11:51:15,385 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:15,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T11:51:15,386 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T11:51:15,388 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740 2024-11-15T11:51:15,391 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740 2024-11-15T11:51:15,395 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T11:51:15,395 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T11:51:15,396 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:51:15,399 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T11:51:15,400 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67874070, jitterRate=0.01140245795249939}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:51:15,401 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T11:51:15,402 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731671475373Writing region info on filesystem at 1731671475373Initializing all the Stores at 1731671475375 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671475375Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671475375Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671475376 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731671475376Cleaning up temporary data from old regions at 1731671475395 (+19 ms)Running coprocessor post-open hooks at 1731671475401 (+6 ms)Region opened successfully at 1731671475402 (+1 ms) 2024-11-15T11:51:15,408 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731671475306 2024-11-15T11:51:15,420 DEBUG [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T11:51:15,420 INFO [RS_OPEN_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T11:51:15,422 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:15,425 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7adf9b3d9d04,44479,1731671471201, state=OPEN 2024-11-15T11:51:15,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T11:51:15,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T11:51:15,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T11:51:15,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T11:51:15,487 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:15,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T11:51:15,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7adf9b3d9d04,44479,1731671471201 in 346 msec 2024-11-15T11:51:15,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T11:51:15,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 569 msec 2024-11-15T11:51:15,503 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T11:51:15,503 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T11:51:15,527 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:15,528 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:15,552 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:15,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:15,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6090 sec 2024-11-15T11:51:15,579 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731671475579, completionTime=-1 2024-11-15T11:51:15,582 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T11:51:15,583 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T11:51:15,608 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T11:51:15,608 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731671535608 2024-11-15T11:51:15,609 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731671595609 2024-11-15T11:51:15,609 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-15T11:51:15,611 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,611 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,611 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,612 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7adf9b3d9d04:36957, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,613 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,613 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,619 DEBUG [master/7adf9b3d9d04:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T11:51:15,638 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.242sec 2024-11-15T11:51:15,639 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T11:51:15,640 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T11:51:15,641 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T11:51:15,642 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T11:51:15,642 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T11:51:15,643 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T11:51:15,643 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T11:51:15,651 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T11:51:15,651 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T11:51:15,652 INFO [master/7adf9b3d9d04:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7adf9b3d9d04,36957,1731671470483-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T11:51:15,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:15,720 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T11:51:15,721 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T11:51:15,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:15,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:15,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:15,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424b8b64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:15,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:15,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:15,747 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:15,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:15,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:15,787 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=10952 2024-11-15T11:51:15,796 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:51:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27423f7d 2024-11-15T11:51:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:51:15,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:51:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:51:15,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:51:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:15,859 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:51:15,862 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T11:51:15,862 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:15,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:51:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741835_1011 (size=1061) 2024-11-15T11:51:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:16,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:51:16,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741836_1012 (size=53) 2024-11-15T11:51:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:16,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:16,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 18293b2c63865e9766ac3df4bea5a42d, disabling compactions & flushes 2024-11-15T11:51:16,694 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:16,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:16,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. after waiting 0 ms 2024-11-15T11:51:16,694 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:16,694 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:16,695 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18293b2c63865e9766ac3df4bea5a42d: Waiting for close lock at 1731671476694Disabling compacts and flushes for region at 1731671476694Disabling writes for close at 1731671476694Writing region close event to WAL at 1731671476694Closed at 1731671476694 2024-11-15T11:51:16,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:51:16,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671476698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671476698"}]},"ts":"1731671476698"} 2024-11-15T11:51:16,709 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:51:16,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:51:16,715 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671476712"}]},"ts":"1731671476712"} 2024-11-15T11:51:16,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:51:16,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, ASSIGN}] 2024-11-15T11:51:16,724 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, ASSIGN 2024-11-15T11:51:16,726 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:51:16,878 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18293b2c63865e9766ac3df4bea5a42d, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:16,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, ASSIGN because future has completed 2024-11-15T11:51:16,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:17,044 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:17,045 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:51:17,045 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,045 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:17,046 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,046 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,049 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,053 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:17,053 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18293b2c63865e9766ac3df4bea5a42d columnFamilyName A 2024-11-15T11:51:17,053 DEBUG [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:17,056 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(327): Store=18293b2c63865e9766ac3df4bea5a42d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:17,057 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,061 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:17,062 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18293b2c63865e9766ac3df4bea5a42d columnFamilyName B 2024-11-15T11:51:17,062 DEBUG [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:17,064 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(327): Store=18293b2c63865e9766ac3df4bea5a42d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:17,064 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,068 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:17,069 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18293b2c63865e9766ac3df4bea5a42d columnFamilyName C 2024-11-15T11:51:17,069 DEBUG [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:17,070 INFO [StoreOpener-18293b2c63865e9766ac3df4bea5a42d-1 {}] regionserver.HStore(327): Store=18293b2c63865e9766ac3df4bea5a42d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:17,071 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:17,071 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,074 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,075 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,079 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,080 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,082 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:51:17,091 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,108 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:51:17,109 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 18293b2c63865e9766ac3df4bea5a42d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67563293, jitterRate=0.006771519780158997}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:51:17,109 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:17,110 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 18293b2c63865e9766ac3df4bea5a42d: Running coprocessor pre-open hook at 1731671477046Writing region info on filesystem at 1731671477046Initializing all the Stores at 1731671477048 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671477048Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671477048Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671477048Cleaning up temporary data from old regions at 1731671477080 (+32 ms)Running coprocessor post-open hooks at 1731671477109 (+29 ms)Region opened successfully at 1731671477110 (+1 ms) 2024-11-15T11:51:17,113 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., pid=6, masterSystemTime=1731671477037 2024-11-15T11:51:17,119 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:17,119 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:17,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18293b2c63865e9766ac3df4bea5a42d, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:17,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:51:17,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T11:51:17,146 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 in 254 msec 2024-11-15T11:51:17,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T11:51:17,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, ASSIGN in 423 msec 2024-11-15T11:51:17,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:51:17,151 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671477151"}]},"ts":"1731671477151"} 2024-11-15T11:51:17,155 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:51:17,157 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:51:17,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3130 sec 2024-11-15T11:51:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T11:51:18,023 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:18,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,035 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,037 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,042 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,045 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:51:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-15T11:51:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:51:18,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:51:18,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,068 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,072 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,074 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,076 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,080 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,083 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,083 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,085 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,088 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1e7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,090 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,092 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b21b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,094 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a8d6bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,099 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c793760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,106 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,113 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a0e39b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,116 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:18,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:18,119 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c994daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:18,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:18,122 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:18,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e186e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:18,129 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,130 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,132 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,134 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,137 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,138 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,143 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,168 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,170 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,182 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:18,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:18,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:51:18,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T11:51:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:18,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:18,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:18,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:18,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:18,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:51:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:18,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:18,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:18,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:18,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:18,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:18,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ed65d5a3a7244a8eab338e5971a92ba4 is 50, key is test_row_0/A:col10/1731671478237/Put/seqid=0 2024-11-15T11:51:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671538345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741837_1013 (size=9657) 2024-11-15T11:51:18,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ed65d5a3a7244a8eab338e5971a92ba4 2024-11-15T11:51:18,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671538366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671538370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671538381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671538381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:18,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/288d2da5c7304e6d96c03327759a3d24 is 50, key is test_row_0/B:col10/1731671478237/Put/seqid=0 2024-11-15T11:51:18,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741838_1014 (size=9657) 2024-11-15T11:51:18,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:18,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671538531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671538533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671538533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:18,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671538755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671538755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671538756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:18,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:18,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/288d2da5c7304e6d96c03327759a3d24 2024-11-15T11:51:18,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:18,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/2a6e23619e374de88ffeb98b5ad5b105 is 50, key is test_row_0/C:col10/1731671478237/Put/seqid=0 2024-11-15T11:51:19,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741839_1015 (size=9657) 2024-11-15T11:51:19,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/2a6e23619e374de88ffeb98b5ad5b105 2024-11-15T11:51:19,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ed65d5a3a7244a8eab338e5971a92ba4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4 2024-11-15T11:51:19,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4, entries=100, sequenceid=13, filesize=9.4 K 2024-11-15T11:51:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/288d2da5c7304e6d96c03327759a3d24 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24 2024-11-15T11:51:19,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24, entries=100, sequenceid=13, filesize=9.4 K 2024-11-15T11:51:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:19,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/2a6e23619e374de88ffeb98b5ad5b105 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105 2024-11-15T11:51:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:19,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:19,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:19,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671539072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671539072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671539074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671539074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671539082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105, entries=100, sequenceid=13, filesize=9.4 K 2024-11-15T11:51:19,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 18293b2c63865e9766ac3df4bea5a42d in 863ms, sequenceid=13, compaction requested=false 2024-11-15T11:51:19,115 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T11:51:19,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:19,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T11:51:19,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:19,239 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:19,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f9eea9157a94163926ff59dde7a58d4 is 50, key is test_row_0/A:col10/1731671478338/Put/seqid=0 2024-11-15T11:51:19,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741840_1016 (size=12001) 2024-11-15T11:51:19,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f9eea9157a94163926ff59dde7a58d4 2024-11-15T11:51:19,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3721e6c7d1c14e74b783c5aaf846187f is 50, key is test_row_0/B:col10/1731671478338/Put/seqid=0 2024-11-15T11:51:19,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:19,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741841_1017 (size=12001) 2024-11-15T11:51:19,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:19,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671539619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671539624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671539624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671539634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671539636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671539742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671539743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671539743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671539751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671539752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,764 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3721e6c7d1c14e74b783c5aaf846187f 2024-11-15T11:51:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/713925d53fdd42818e85e455ce173b9f is 50, key is test_row_0/C:col10/1731671478338/Put/seqid=0 2024-11-15T11:51:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741842_1018 (size=12001) 2024-11-15T11:51:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671539951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671539964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671539965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671539965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671539965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,186 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T11:51:20,244 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/713925d53fdd42818e85e455ce173b9f 2024-11-15T11:51:20,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f9eea9157a94163926ff59dde7a58d4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4 2024-11-15T11:51:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671540271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T11:51:20,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3721e6c7d1c14e74b783c5aaf846187f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f 2024-11-15T11:51:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671540283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671540284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671540285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671540286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,317 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T11:51:20,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/713925d53fdd42818e85e455ce173b9f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f 2024-11-15T11:51:20,342 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-15T11:51:20,346 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 18293b2c63865e9766ac3df4bea5a42d in 1108ms, sequenceid=33, compaction requested=false 2024-11-15T11:51:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:20,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T11:51:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T11:51:20,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T11:51:20,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1310 sec 2024-11-15T11:51:20,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.1530 sec 2024-11-15T11:51:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T11:51:20,373 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:20,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T11:51:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:20,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:20,381 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:20,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:20,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T11:51:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:20,537 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/cbef026738ea49b3881b811a80d54747 is 50, key is test_row_0/A:col10/1731671479628/Put/seqid=0 2024-11-15T11:51:20,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741843_1019 (size=12001) 2024-11-15T11:51:20,588 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/cbef026738ea49b3881b811a80d54747 2024-11-15T11:51:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c96d88df734461aa51f515e5ba914fc is 50, key is test_row_0/B:col10/1731671479628/Put/seqid=0 2024-11-15T11:51:20,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741844_1020 (size=12001) 2024-11-15T11:51:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:20,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671540847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671540847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671540851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671540854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671540862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,885 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:51:20,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671540972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,977 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T11:51:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,978 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T11:51:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671540973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671540973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671540973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671540973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:20,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T11:51:20,982 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:20,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T11:51:20,982 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T11:51:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:20,984 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T11:51:20,984 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T11:51:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:21,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c96d88df734461aa51f515e5ba914fc 2024-11-15T11:51:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/679218196eaa4ad3a580c9dc2053d25c is 50, key is test_row_0/C:col10/1731671479628/Put/seqid=0 2024-11-15T11:51:21,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741845_1021 (size=12001) 2024-11-15T11:51:21,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671541191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671541192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671541194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671541194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671541195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671541501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671541502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671541502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671541501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671541503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:21,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:21,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/679218196eaa4ad3a580c9dc2053d25c 2024-11-15T11:51:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/cbef026738ea49b3881b811a80d54747 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747 2024-11-15T11:51:21,562 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747, entries=150, sequenceid=45, filesize=11.7 K 2024-11-15T11:51:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c96d88df734461aa51f515e5ba914fc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc 2024-11-15T11:51:21,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc, entries=150, sequenceid=45, filesize=11.7 K 2024-11-15T11:51:21,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/679218196eaa4ad3a580c9dc2053d25c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c 2024-11-15T11:51:21,611 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c, entries=150, sequenceid=45, filesize=11.7 K 2024-11-15T11:51:21,614 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 18293b2c63865e9766ac3df4bea5a42d in 1077ms, sequenceid=45, compaction requested=true 2024-11-15T11:51:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T11:51:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T11:51:21,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T11:51:21,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2450 sec 2024-11-15T11:51:21,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2580 sec 2024-11-15T11:51:22,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671480845, max timestamp=1731671482024Num uniques 859; 2024-11-15T11:51:22,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:51:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:22,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=102.87 KB heapSize=301.95 KB 2024-11-15T11:51:22,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:22,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671480845, max timestamp=1731671482027Num uniques 859; 2024-11-15T11:51:22,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:51:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:22,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/aa741c0586bb463e9866bfa77f5dad19 is 50, key is test_row_0/A:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741846_1022 (size=14341) 2024-11-15T11:51:22,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/aa741c0586bb463e9866bfa77f5dad19 2024-11-15T11:51:22,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/4103630ec6da4b88adf612a63e072749 is 50, key is test_row_0/B:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741847_1023 (size=12001) 2024-11-15T11:51:22,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/4103630ec6da4b88adf612a63e072749 2024-11-15T11:51:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671542098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671542116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671542120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671542125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671542128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/93e0b0dc217246e8b9e53672f3450e62 is 50, key is test_row_0/C:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,142 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741848_1024 (size=12001) 2024-11-15T11:51:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671542241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671542241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671542241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671542253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671542254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671542461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671542461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671542461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671542471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671542471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T11:51:22,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:22,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T11:51:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T11:51:22,547 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:22,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:22,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:22,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/93e0b0dc217246e8b9e53672f3450e62 2024-11-15T11:51:22,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/aa741c0586bb463e9866bfa77f5dad19 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19 2024-11-15T11:51:22,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19, entries=200, sequenceid=69, filesize=14.0 K 2024-11-15T11:51:22,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/4103630ec6da4b88adf612a63e072749 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749 2024-11-15T11:51:22,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749, entries=150, sequenceid=69, filesize=11.7 K 2024-11-15T11:51:22,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/93e0b0dc217246e8b9e53672f3450e62 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62 2024-11-15T11:51:22,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62, entries=150, sequenceid=69, filesize=11.7 K 2024-11-15T11:51:22,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~223.89 KB/229264, currentSize=100.63 KB/103050 for 18293b2c63865e9766ac3df4bea5a42d in 608ms, sequenceid=69, compaction requested=true 2024-11-15T11:51:22,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T11:51:22,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:22,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:22,664 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:22,671 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:22,675 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:22,676 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:22,678 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:22,678 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:22,678 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:22,678 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:22,678 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=44.6 K 2024-11-15T11:51:22,678 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=46.9 K 2024-11-15T11:51:22,680 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed65d5a3a7244a8eab338e5971a92ba4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671478223 2024-11-15T11:51:22,680 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 288d2da5c7304e6d96c03327759a3d24, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671478223 2024-11-15T11:51:22,683 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f9eea9157a94163926ff59dde7a58d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731671478338 2024-11-15T11:51:22,683 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3721e6c7d1c14e74b783c5aaf846187f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731671478338 2024-11-15T11:51:22,685 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbef026738ea49b3881b811a80d54747, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731671479618 2024-11-15T11:51:22,686 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa741c0586bb463e9866bfa77f5dad19, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:22,691 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c96d88df734461aa51f515e5ba914fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731671479618 2024-11-15T11:51:22,692 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4103630ec6da4b88adf612a63e072749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:22,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T11:51:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:22,707 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:22,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:22,726 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:22,727 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#12 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:22,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/15fb9db34bf4446e97f07eb867c25028 is 50, key is test_row_0/A:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,728 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/092a5912c7d44f688c8ae8f578bc1979 is 50, key is test_row_0/B:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/82dd094efb9b4a48978fade9046cdc85 is 50, key is test_row_0/A:col10/1731671482124/Put/seqid=0 2024-11-15T11:51:22,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741849_1025 (size=12139) 2024-11-15T11:51:22,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:22,783 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/092a5912c7d44f688c8ae8f578bc1979 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/092a5912c7d44f688c8ae8f578bc1979 2024-11-15T11:51:22,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741850_1026 (size=12139) 2024-11-15T11:51:22,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741851_1027 (size=12001) 2024-11-15T11:51:22,809 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/82dd094efb9b4a48978fade9046cdc85 2024-11-15T11:51:22,812 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/15fb9db34bf4446e97f07eb867c25028 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/15fb9db34bf4446e97f07eb867c25028 2024-11-15T11:51:22,823 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into 092a5912c7d44f688c8ae8f578bc1979(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:22,823 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:22,830 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=12, startTime=1731671482661; duration=0sec 2024-11-15T11:51:22,831 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:22,831 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:22,831 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:22,833 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into 15fb9db34bf4446e97f07eb867c25028(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:22,833 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:22,833 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=12, startTime=1731671482648; duration=0sec 2024-11-15T11:51:22,833 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:22,833 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:22,835 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:22,835 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:22,835 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:22,835 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=44.6 K 2024-11-15T11:51:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671542821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,839 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6e23619e374de88ffeb98b5ad5b105, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671478223 2024-11-15T11:51:22,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f5dc11fb69d944f4a2fff8d235c7a0b5 is 50, key is test_row_0/B:col10/1731671482124/Put/seqid=0 2024-11-15T11:51:22,842 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 713925d53fdd42818e85e455ce173b9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731671478338 2024-11-15T11:51:22,844 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 679218196eaa4ad3a580c9dc2053d25c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731671479618 2024-11-15T11:51:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671542822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671542831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,845 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 93e0b0dc217246e8b9e53672f3450e62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671542831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671542836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T11:51:22,883 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#16 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:22,884 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/37449478fdf24fa1b94acf0dbfab7393 is 50, key is test_row_0/C:col10/1731671482027/Put/seqid=0 2024-11-15T11:51:22,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741852_1028 (size=12001) 2024-11-15T11:51:22,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f5dc11fb69d944f4a2fff8d235c7a0b5 2024-11-15T11:51:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741853_1029 (size=12139) 2024-11-15T11:51:22,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/8994dfcd774542c98cbec42013103b46 is 50, key is test_row_0/C:col10/1731671482124/Put/seqid=0 2024-11-15T11:51:22,930 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/37449478fdf24fa1b94acf0dbfab7393 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/37449478fdf24fa1b94acf0dbfab7393 2024-11-15T11:51:22,947 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 37449478fdf24fa1b94acf0dbfab7393(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:22,947 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:22,947 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=12, startTime=1731671482661; duration=0sec 2024-11-15T11:51:22,947 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:22,947 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:22,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671542941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741854_1030 (size=12001) 2024-11-15T11:51:22,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/8994dfcd774542c98cbec42013103b46 2024-11-15T11:51:22,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671542950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671542951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671542962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671542962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/82dd094efb9b4a48978fade9046cdc85 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85 2024-11-15T11:51:22,984 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T11:51:22,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f5dc11fb69d944f4a2fff8d235c7a0b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5 2024-11-15T11:51:23,000 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T11:51:23,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/8994dfcd774542c98cbec42013103b46 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46 2024-11-15T11:51:23,017 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46, entries=150, sequenceid=87, filesize=11.7 K 2024-11-15T11:51:23,019 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 18293b2c63865e9766ac3df4bea5a42d in 311ms, sequenceid=87, compaction requested=false 2024-11-15T11:51:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T11:51:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T11:51:23,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T11:51:23,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 470 msec 2024-11-15T11:51:23,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 490 msec 2024-11-15T11:51:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:23,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:23,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T11:51:23,175 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:23,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T11:51:23,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:23,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:23,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:23,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8bcb92efece449c6a931180f6930b99a is 50, key is test_row_0/A:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671543203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671543205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671543207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671543210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671543207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741855_1031 (size=14341) 2024-11-15T11:51:23,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8bcb92efece449c6a931180f6930b99a 2024-11-15T11:51:23,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/273c085c206e4f67b99997a338355d1a is 50, key is test_row_0/B:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741856_1032 (size=12001) 2024-11-15T11:51:23,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/273c085c206e4f67b99997a338355d1a 2024-11-15T11:51:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:23,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/5e2144c28d7d495587de1ccf3e21c227 is 50, key is test_row_0/C:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671543322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671543331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671543331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T11:51:23,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741857_1033 (size=12001) 2024-11-15T11:51:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:23,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671543334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/5e2144c28d7d495587de1ccf3e21c227 2024-11-15T11:51:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671543334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:23,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8bcb92efece449c6a931180f6930b99a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a 2024-11-15T11:51:23,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a, entries=200, sequenceid=105, filesize=14.0 K 2024-11-15T11:51:23,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/273c085c206e4f67b99997a338355d1a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a 2024-11-15T11:51:23,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T11:51:23,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/5e2144c28d7d495587de1ccf3e21c227 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227 2024-11-15T11:51:23,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227, entries=150, sequenceid=105, filesize=11.7 K 2024-11-15T11:51:23,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 260ms, sequenceid=105, compaction requested=true 2024-11-15T11:51:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:23,429 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:23,430 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:23,431 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:23,432 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:23,432 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,432 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:23,432 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:23,432 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/15fb9db34bf4446e97f07eb867c25028, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=37.6 K 2024-11-15T11:51:23,432 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,432 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/092a5912c7d44f688c8ae8f578bc1979, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=35.3 K 2024-11-15T11:51:23,433 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15fb9db34bf4446e97f07eb867c25028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:23,433 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 092a5912c7d44f688c8ae8f578bc1979, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:23,433 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82dd094efb9b4a48978fade9046cdc85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731671482115 2024-11-15T11:51:23,434 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f5dc11fb69d944f4a2fff8d235c7a0b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731671482115 2024-11-15T11:51:23,434 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcb92efece449c6a931180f6930b99a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:23,434 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 273c085c206e4f67b99997a338355d1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:23,452 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:23,453 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/345b5b5dc4c4446e9ddbedb2a97ea459 is 50, key is test_row_0/B:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,460 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:23,461 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/e0b2eafde3c24c17bfbc1524137841da is 50, key is test_row_0/A:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741858_1034 (size=12241) 2024-11-15T11:51:23,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741859_1035 (size=12241) 2024-11-15T11:51:23,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T11:51:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T11:51:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:23,506 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/345b5b5dc4c4446e9ddbedb2a97ea459 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/345b5b5dc4c4446e9ddbedb2a97ea459 2024-11-15T11:51:23,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/0bff829d67c04f4ea5732da6fed7b740 is 50, key is test_row_0/A:col10/1731671483197/Put/seqid=0 2024-11-15T11:51:23,522 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into 345b5b5dc4c4446e9ddbedb2a97ea459(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:23,522 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:23,522 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=13, startTime=1731671483429; duration=0sec 2024-11-15T11:51:23,523 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:23,523 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:23,523 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:23,534 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:23,534 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:23,534 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:23,534 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/37449478fdf24fa1b94acf0dbfab7393, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=35.3 K 2024-11-15T11:51:23,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741860_1036 (size=12001) 2024-11-15T11:51:23,537 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/0bff829d67c04f4ea5732da6fed7b740 2024-11-15T11:51:23,538 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 37449478fdf24fa1b94acf0dbfab7393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731671480849 2024-11-15T11:51:23,539 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8994dfcd774542c98cbec42013103b46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731671482115 2024-11-15T11:51:23,545 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e2144c28d7d495587de1ccf3e21c227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:23,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:23,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/1b14175c7089400cacc621f23126d82b is 50, key is test_row_0/B:col10/1731671483197/Put/seqid=0 2024-11-15T11:51:23,589 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:23,590 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/d32ed07a9af84300b728c613b1064140 is 50, key is test_row_0/C:col10/1731671482774/Put/seqid=0 2024-11-15T11:51:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671543577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671543579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671543581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671543597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671543601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741862_1038 (size=12241) 2024-11-15T11:51:23,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741861_1037 (size=12001) 2024-11-15T11:51:23,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671543700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671543701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671543702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671543710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671543722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:23,909 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/e0b2eafde3c24c17bfbc1524137841da as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e0b2eafde3c24c17bfbc1524137841da 2024-11-15T11:51:23,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671543911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671543911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671543912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671543921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:23,927 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into e0b2eafde3c24c17bfbc1524137841da(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:23,928 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:23,928 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=13, startTime=1731671483429; duration=0sec 2024-11-15T11:51:23,928 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:23,928 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:23,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:23,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671543931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:23,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,016 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/1b14175c7089400cacc621f23126d82b 2024-11-15T11:51:24,035 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/d32ed07a9af84300b728c613b1064140 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d32ed07a9af84300b728c613b1064140 2024-11-15T11:51:24,049 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into d32ed07a9af84300b728c613b1064140(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:24,049 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:24,049 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=13, startTime=1731671483429; duration=0sec 2024-11-15T11:51:24,049 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:24,049 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:24,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/58b504aa2a974e4083fd778faa45159b is 50, key is test_row_0/C:col10/1731671483197/Put/seqid=0 2024-11-15T11:51:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741863_1039 (size=12001) 2024-11-15T11:51:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671544232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671544232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671544232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671544232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671544251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:24,476 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/58b504aa2a974e4083fd778faa45159b 2024-11-15T11:51:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/0bff829d67c04f4ea5732da6fed7b740 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740 2024-11-15T11:51:24,503 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T11:51:24,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/1b14175c7089400cacc621f23126d82b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b 2024-11-15T11:51:24,521 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T11:51:24,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/58b504aa2a974e4083fd778faa45159b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b 2024-11-15T11:51:24,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b, entries=150, sequenceid=122, filesize=11.7 K 2024-11-15T11:51:24,542 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 18293b2c63865e9766ac3df4bea5a42d in 1043ms, sequenceid=122, compaction requested=false 2024-11-15T11:51:24,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:24,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:24,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T11:51:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T11:51:24,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T11:51:24,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3600 sec 2024-11-15T11:51:24,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3750 sec 2024-11-15T11:51:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:24,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:24,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:24,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f14013bacee40bfa451b2df471d9a01 is 50, key is test_row_0/A:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671544769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671544771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671544774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671544778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671544779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741864_1040 (size=12151) 2024-11-15T11:51:24,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671544880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671544890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671544891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671544891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:24,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671545090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671545101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671545101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671545101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f14013bacee40bfa451b2df471d9a01 2024-11-15T11:51:25,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3b7d6e2a589f46bdad6592980cb00c9c is 50, key is test_row_0/B:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:25,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741865_1041 (size=12151) 2024-11-15T11:51:25,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3b7d6e2a589f46bdad6592980cb00c9c 2024-11-15T11:51:25,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/6cbe17d4440b4ceaae2484719ef44b31 is 50, key is test_row_0/C:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:25,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741866_1042 (size=12151) 2024-11-15T11:51:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T11:51:25,340 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:25,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T11:51:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:25,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:25,346 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:25,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671545401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671545410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671545410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671545410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:25,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:25,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/6cbe17d4440b4ceaae2484719ef44b31 2024-11-15T11:51:25,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8f14013bacee40bfa451b2df471d9a01 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01 2024-11-15T11:51:25,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T11:51:25,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/3b7d6e2a589f46bdad6592980cb00c9c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c 2024-11-15T11:51:25,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T11:51:25,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/6cbe17d4440b4ceaae2484719ef44b31 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31 2024-11-15T11:51:25,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T11:51:25,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 18293b2c63865e9766ac3df4bea5a42d in 960ms, sequenceid=142, compaction requested=true 2024-11-15T11:51:25,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:25,707 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:25,707 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:25,708 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:25,709 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:25,709 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,709 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e0b2eafde3c24c17bfbc1524137841da, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=35.5 K 2024-11-15T11:51:25,709 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:25,709 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:25,709 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,709 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0b2eafde3c24c17bfbc1524137841da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:25,709 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/345b5b5dc4c4446e9ddbedb2a97ea459, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=35.5 K 2024-11-15T11:51:25,710 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 345b5b5dc4c4446e9ddbedb2a97ea459, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:25,710 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bff829d67c04f4ea5732da6fed7b740, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731671483197 2024-11-15T11:51:25,711 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b14175c7089400cacc621f23126d82b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731671483197 2024-11-15T11:51:25,711 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f14013bacee40bfa451b2df471d9a01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:25,711 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7d6e2a589f46bdad6592980cb00c9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:25,730 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:25,734 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/e8c9ede5f35b459f9fdce8b37a454db4 is 50, key is test_row_0/A:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:25,737 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:25,737 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/683a747f6ff3482a91a75104978e0239 is 50, key is test_row_0/B:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741868_1044 (size=12493) 2024-11-15T11:51:25,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741867_1043 (size=12493) 2024-11-15T11:51:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:25,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T11:51:25,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:25,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:25,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:25,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:25,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:25,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:25,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:25,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/16515791cef5405ca4797ad21c7366ea is 50, key is test_row_0/A:col10/1731671484775/Put/seqid=0 2024-11-15T11:51:25,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741869_1045 (size=14541) 2024-11-15T11:51:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671545900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671545911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671545921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671545922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671545931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:25,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:25,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:25,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:25,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:26,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671546010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,166 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/683a747f6ff3482a91a75104978e0239 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/683a747f6ff3482a91a75104978e0239 2024-11-15T11:51:26,169 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/e8c9ede5f35b459f9fdce8b37a454db4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e8c9ede5f35b459f9fdce8b37a454db4 2024-11-15T11:51:26,180 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into e8c9ede5f35b459f9fdce8b37a454db4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:26,180 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:26,180 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=13, startTime=1731671485706; duration=0sec 2024-11-15T11:51:26,180 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:26,180 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:26,180 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:26,181 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into 683a747f6ff3482a91a75104978e0239(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:26,181 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:26,181 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=13, startTime=1731671485707; duration=0sec 2024-11-15T11:51:26,182 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:26,182 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:26,182 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:26,182 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:26,182 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,182 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d32ed07a9af84300b728c613b1064140, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=35.5 K 2024-11-15T11:51:26,183 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32ed07a9af84300b728c613b1064140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731671482774 2024-11-15T11:51:26,183 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b504aa2a974e4083fd778faa45159b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731671483197 2024-11-15T11:51:26,184 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cbe17d4440b4ceaae2484719ef44b31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:26,198 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:26,198 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/b6ddd234566947dfb09d90f966c46861 is 50, key is test_row_0/C:col10/1731671483592/Put/seqid=0 2024-11-15T11:51:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671546220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741870_1046 (size=12493) 2024-11-15T11:51:26,240 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/b6ddd234566947dfb09d90f966c46861 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b6ddd234566947dfb09d90f966c46861 2024-11-15T11:51:26,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/16515791cef5405ca4797ad21c7366ea 2024-11-15T11:51:26,251 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into b6ddd234566947dfb09d90f966c46861(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:26,251 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:26,251 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=13, startTime=1731671485707; duration=0sec 2024-11-15T11:51:26,251 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:26,251 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:26,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2688a9980c524d6594f273a142965d16 is 50, key is test_row_0/B:col10/1731671484775/Put/seqid=0 2024-11-15T11:51:26,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741871_1047 (size=12151) 2024-11-15T11:51:26,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:26,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671546531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2688a9980c524d6594f273a142965d16 2024-11-15T11:51:26,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/4cab028948164f1ba9ff5cb04810cf95 is 50, key is test_row_0/C:col10/1731671484775/Put/seqid=0 2024-11-15T11:51:26,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:26,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741872_1048 (size=12151) 2024-11-15T11:51:26,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/4cab028948164f1ba9ff5cb04810cf95 2024-11-15T11:51:26,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/16515791cef5405ca4797ad21c7366ea as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea 2024-11-15T11:51:26,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea, entries=200, sequenceid=158, filesize=14.2 K 2024-11-15T11:51:26,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2688a9980c524d6594f273a142965d16 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16 2024-11-15T11:51:26,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T11:51:26,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/4cab028948164f1ba9ff5cb04810cf95 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95 2024-11-15T11:51:26,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T11:51:26,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 18293b2c63865e9766ac3df4bea5a42d in 980ms, sequenceid=158, compaction requested=false 2024-11-15T11:51:26,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:26,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T11:51:26,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:26,891 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:26,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/f7bfc96411004d5b9314d651b9065352 is 50, key is test_row_0/A:col10/1731671485897/Put/seqid=0 2024-11-15T11:51:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741873_1049 (size=12151) 2024-11-15T11:51:26,913 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/f7bfc96411004d5b9314d651b9065352 2024-11-15T11:51:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/6d3caebd60214d1181eaaaea56854019 is 50, key is test_row_0/B:col10/1731671485897/Put/seqid=0 2024-11-15T11:51:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:26,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741874_1050 (size=12151) 2024-11-15T11:51:26,948 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/6d3caebd60214d1181eaaaea56854019 2024-11-15T11:51:26,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671546965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671546967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/80391f371af8416d88d2efcfc6df6be2 is 50, key is test_row_0/C:col10/1731671485897/Put/seqid=0 2024-11-15T11:51:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671546973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671546973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:26,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741875_1051 (size=12151) 2024-11-15T11:51:27,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/80391f371af8416d88d2efcfc6df6be2 2024-11-15T11:51:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/f7bfc96411004d5b9314d651b9065352 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352 2024-11-15T11:51:27,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:51:27,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/6d3caebd60214d1181eaaaea56854019 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019 2024-11-15T11:51:27,040 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:51:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/80391f371af8416d88d2efcfc6df6be2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2 2024-11-15T11:51:27,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671547041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,054 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:51:27,055 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 164ms, sequenceid=177, compaction requested=true 2024-11-15T11:51:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T11:51:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T11:51:27,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T11:51:27,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-15T11:51:27,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7200 sec 2024-11-15T11:51:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:27,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:27,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:27,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:27,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:27,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:27,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/5701e353732d41d5b0a3dd8d44a8b77c is 50, key is test_row_0/A:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:27,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741876_1052 (size=14541) 2024-11-15T11:51:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671547105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671547107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671547110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671547110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671547221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671547222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671547223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671547223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671547430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671547430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671547430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671547431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T11:51:27,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:27,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T11:51:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:27,495 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:27,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:27,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:27,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/5701e353732d41d5b0a3dd8d44a8b77c 2024-11-15T11:51:27,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/73058b7bd2614628a5989d6a6598cb8d is 50, key is test_row_0/B:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:27,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741877_1053 (size=12151) 2024-11-15T11:51:27,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/73058b7bd2614628a5989d6a6598cb8d 2024-11-15T11:51:27,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/591c892e38814fe5b4b74438e0679021 is 50, key is test_row_0/C:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:27,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741878_1054 (size=12151) 2024-11-15T11:51:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:27,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:27,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671547740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671547741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671547741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671547741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:27,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:27,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:27,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:27,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/591c892e38814fe5b4b74438e0679021 2024-11-15T11:51:27,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:27,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:27,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:27,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/5701e353732d41d5b0a3dd8d44a8b77c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c 2024-11-15T11:51:27,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c, entries=200, sequenceid=195, filesize=14.2 K 2024-11-15T11:51:27,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/73058b7bd2614628a5989d6a6598cb8d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d 2024-11-15T11:51:27,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:51:27,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/591c892e38814fe5b4b74438e0679021 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021 2024-11-15T11:51:28,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:51:28,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 18293b2c63865e9766ac3df4bea5a42d in 918ms, sequenceid=195, compaction requested=true 2024-11-15T11:51:28,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,003 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:28,003 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:28,005 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:28,006 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:28,006 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,006 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e8c9ede5f35b459f9fdce8b37a454db4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=52.5 K 2024-11-15T11:51:28,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:28,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:28,007 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,007 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/683a747f6ff3482a91a75104978e0239, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=47.8 K 2024-11-15T11:51:28,007 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c9ede5f35b459f9fdce8b37a454db4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:28,008 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 683a747f6ff3482a91a75104978e0239, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:28,008 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16515791cef5405ca4797ad21c7366ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671484757 2024-11-15T11:51:28,008 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2688a9980c524d6594f273a142965d16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671484757 2024-11-15T11:51:28,008 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7bfc96411004d5b9314d651b9065352, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671485861 2024-11-15T11:51:28,009 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3caebd60214d1181eaaaea56854019, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671485861 2024-11-15T11:51:28,009 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5701e353732d41d5b0a3dd8d44a8b77c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,009 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 73058b7bd2614628a5989d6a6598cb8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,030 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,031 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/ab88a955dc5c4ed09f510d45b18cc891 is 50, key is test_row_0/B:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:28,033 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,034 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/631f31d3d481481bbc65a2516b4c3683 is 50, key is test_row_0/A:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:28,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741879_1055 (size=12629) 2024-11-15T11:51:28,057 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/ab88a955dc5c4ed09f510d45b18cc891 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ab88a955dc5c4ed09f510d45b18cc891 2024-11-15T11:51:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:28,069 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into ab88a955dc5c4ed09f510d45b18cc891(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:28,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T11:51:28,069 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,070 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=12, startTime=1731671488003; duration=0sec 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:28,070 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,070 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:28,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,070 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741880_1056 (size=12629) 2024-11-15T11:51:28,076 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:28,076 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:28,076 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,076 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b6ddd234566947dfb09d90f966c46861, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=47.8 K 2024-11-15T11:51:28,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/466a54fac09b43d093c4d8f376bcf4fe is 50, key is test_row_0/A:col10/1731671488067/Put/seqid=0 2024-11-15T11:51:28,080 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ddd234566947dfb09d90f966c46861, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731671483576 2024-11-15T11:51:28,081 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cab028948164f1ba9ff5cb04810cf95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671484757 2024-11-15T11:51:28,085 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 80391f371af8416d88d2efcfc6df6be2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671485861 2024-11-15T11:51:28,087 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 591c892e38814fe5b4b74438e0679021, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,090 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/631f31d3d481481bbc65a2516b4c3683 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/631f31d3d481481bbc65a2516b4c3683 2024-11-15T11:51:28,110 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into 631f31d3d481481bbc65a2516b4c3683(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:28,110 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,110 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=12, startTime=1731671488002; duration=0sec 2024-11-15T11:51:28,110 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,110 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:28,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741881_1057 (size=14541) 2024-11-15T11:51:28,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/466a54fac09b43d093c4d8f376bcf4fe 2024-11-15T11:51:28,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:28,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,117 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#45 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,118 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/68d8ccc1eec34551b716f08fd3c14ef5 is 50, key is test_row_0/C:col10/1731671487083/Put/seqid=0 2024-11-15T11:51:28,119 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36957 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=7adf9b3d9d04,44479,1731671471201, table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T11:51:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:28,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/64df7efd470f454b8e02b65f5788ec8d is 50, key is test_row_0/B:col10/1731671488067/Put/seqid=0 2024-11-15T11:51:28,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741882_1058 (size=12629) 2024-11-15T11:51:28,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741883_1059 (size=12151) 2024-11-15T11:51:28,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/64df7efd470f454b8e02b65f5788ec8d 2024-11-15T11:51:28,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/3a69e0e9fc984d3d878a5bf3002e3213 is 50, key is test_row_0/C:col10/1731671488067/Put/seqid=0 2024-11-15T11:51:28,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671548191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741884_1060 (size=12151) 2024-11-15T11:51:28,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/3a69e0e9fc984d3d878a5bf3002e3213 2024-11-15T11:51:28,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/466a54fac09b43d093c4d8f376bcf4fe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe 2024-11-15T11:51:28,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe, entries=200, sequenceid=211, filesize=14.2 K 2024-11-15T11:51:28,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/64df7efd470f454b8e02b65f5788ec8d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d 2024-11-15T11:51:28,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T11:51:28,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/3a69e0e9fc984d3d878a5bf3002e3213 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213 2024-11-15T11:51:28,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213, entries=150, sequenceid=211, filesize=11.9 K 2024-11-15T11:51:28,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 178ms, sequenceid=211, compaction requested=false 2024-11-15T11:51:28,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:28,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:28,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/c9e72b4e5f544e61b81ec87d875aedbb is 50, key is test_row_0/A:col10/1731671488253/Put/seqid=0 2024-11-15T11:51:28,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671548275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671548278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671548278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671548275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741885_1061 (size=16931) 2024-11-15T11:51:28,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/c9e72b4e5f544e61b81ec87d875aedbb 2024-11-15T11:51:28,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671548301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c6209c9c83f409ea0ab72883722664d is 50, key is test_row_0/B:col10/1731671488253/Put/seqid=0 2024-11-15T11:51:28,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741886_1062 (size=12151) 2024-11-15T11:51:28,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c6209c9c83f409ea0ab72883722664d 2024-11-15T11:51:28,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/dd2ff62e1ffc4cd0adcab4c02cee293c is 50, key is test_row_0/C:col10/1731671488253/Put/seqid=0 2024-11-15T11:51:28,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741887_1063 (size=12151) 2024-11-15T11:51:28,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/dd2ff62e1ffc4cd0adcab4c02cee293c 2024-11-15T11:51:28,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/c9e72b4e5f544e61b81ec87d875aedbb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb 2024-11-15T11:51:28,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb, entries=250, sequenceid=231, filesize=16.5 K 2024-11-15T11:51:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0c6209c9c83f409ea0ab72883722664d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d 2024-11-15T11:51:28,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T11:51:28,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/dd2ff62e1ffc4cd0adcab4c02cee293c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c 2024-11-15T11:51:28,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T11:51:28,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 18293b2c63865e9766ac3df4bea5a42d in 135ms, sequenceid=231, compaction requested=true 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:28,389 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:28,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:28,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:28,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:28,395 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,395 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/631f31d3d481481bbc65a2516b4c3683, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=43.1 K 2024-11-15T11:51:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:28,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T11:51:28,396 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631f31d3d481481bbc65a2516b4c3683, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:28,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,398 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466a54fac09b43d093c4d8f376bcf4fe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731671487104 2024-11-15T11:51:28,398 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9e72b4e5f544e61b81ec87d875aedbb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731671488164 2024-11-15T11:51:28,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/df3b4083ad924c3096d1a9ec165de08c is 50, key is test_row_0/A:col10/1731671488395/Put/seqid=0 2024-11-15T11:51:28,423 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,424 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/de5d934228844749985549031b9c8ff8 is 50, key is test_row_0/A:col10/1731671488253/Put/seqid=0 2024-11-15T11:51:28,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671548431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671548434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671548436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671548441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741888_1064 (size=12151) 2024-11-15T11:51:28,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/df3b4083ad924c3096d1a9ec165de08c 2024-11-15T11:51:28,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741889_1065 (size=12731) 2024-11-15T11:51:28,499 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/de5d934228844749985549031b9c8ff8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/de5d934228844749985549031b9c8ff8 2024-11-15T11:51:28,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/9676c06cfc5c4736916e95263541c0f2 is 50, key is test_row_0/B:col10/1731671488395/Put/seqid=0 2024-11-15T11:51:28,510 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into de5d934228844749985549031b9c8ff8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:28,510 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,510 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=13, startTime=1731671488389; duration=0sec 2024-11-15T11:51:28,510 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:28,510 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:28,511 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:28,514 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:28,515 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:28,515 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:28,515 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:28,515 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:28,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671548510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:28,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:28,518 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,518 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ab88a955dc5c4ed09f510d45b18cc891, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=36.1 K 2024-11-15T11:51:28,518 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab88a955dc5c4ed09f510d45b18cc891, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,519 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64df7efd470f454b8e02b65f5788ec8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731671487104 2024-11-15T11:51:28,519 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c6209c9c83f409ea0ab72883722664d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731671488181 2024-11-15T11:51:28,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741890_1066 (size=12151) 2024-11-15T11:51:28,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/9676c06cfc5c4736916e95263541c0f2 2024-11-15T11:51:28,547 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,548 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/c01ddda64caf4f30971210db0d03b0b5 is 50, key is test_row_0/B:col10/1731671488253/Put/seqid=0 2024-11-15T11:51:28,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671548550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671548551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671548551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671548551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/d86843f091a1468f80d4e18c0a949f8e is 50, key is test_row_0/C:col10/1731671488395/Put/seqid=0 2024-11-15T11:51:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,566 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/68d8ccc1eec34551b716f08fd3c14ef5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/68d8ccc1eec34551b716f08fd3c14ef5 2024-11-15T11:51:28,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:28,584 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 68d8ccc1eec34551b716f08fd3c14ef5(size=12.3 K), total size for store is 36.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:28,584 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,584 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=12, startTime=1731671488003; duration=0sec 2024-11-15T11:51:28,585 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,585 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:28,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741891_1067 (size=12731) 2024-11-15T11:51:28,599 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/c01ddda64caf4f30971210db0d03b0b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c01ddda64caf4f30971210db0d03b0b5 2024-11-15T11:51:28,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741892_1068 (size=12151) 2024-11-15T11:51:28,610 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into c01ddda64caf4f30971210db0d03b0b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:28,610 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,610 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=13, startTime=1731671488389; duration=0sec 2024-11-15T11:51:28,611 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,611 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:28,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/d86843f091a1468f80d4e18c0a949f8e 2024-11-15T11:51:28,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/df3b4083ad924c3096d1a9ec165de08c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c 2024-11-15T11:51:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:28,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T11:51:28,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/9676c06cfc5c4736916e95263541c0f2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2 2024-11-15T11:51:28,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T11:51:28,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/d86843f091a1468f80d4e18c0a949f8e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e 2024-11-15T11:51:28,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T11:51:28,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 251ms, sequenceid=246, compaction requested=true 2024-11-15T11:51:28,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:28,651 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:28,652 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:28,652 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:28,652 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:28,652 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:28,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 0 2024-11-15T11:51:28,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,652 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:28,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:28,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,653 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:28,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:28,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:28,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:28,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:28,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:28,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:28,655 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:28,655 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:28,655 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,655 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/68d8ccc1eec34551b716f08fd3c14ef5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=47.9 K 2024-11-15T11:51:28,656 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d8ccc1eec34551b716f08fd3c14ef5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671486964 2024-11-15T11:51:28,656 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a69e0e9fc984d3d878a5bf3002e3213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731671487104 2024-11-15T11:51:28,657 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2ff62e1ffc4cd0adcab4c02cee293c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731671488181 2024-11-15T11:51:28,657 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d86843f091a1468f80d4e18c0a949f8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731671488272 2024-11-15T11:51:28,675 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#56 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:28,675 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/4769a00a9cb14d5fbfdb9538a758c81a is 50, key is test_row_0/C:col10/1731671488395/Put/seqid=0 2024-11-15T11:51:28,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741893_1069 (size=12765) 2024-11-15T11:51:28,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T11:51:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:28,733 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T11:51:28,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/61c4e2221af44575badd10d0ca55fb9a is 50, key is test_row_0/A:col10/1731671488433/Put/seqid=0 2024-11-15T11:51:28,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741894_1070 (size=12301) 2024-11-15T11:51:28,757 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/61c4e2221af44575badd10d0ca55fb9a 2024-11-15T11:51:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:28,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:28,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/fdab677889244778b3b5d151404ee2a7 is 50, key is test_row_0/B:col10/1731671488433/Put/seqid=0 2024-11-15T11:51:28,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741895_1071 (size=12301) 2024-11-15T11:51:28,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671548791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671548796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671548796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671548797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671548821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671548901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671548910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671548910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671548910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:28,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,086 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/4769a00a9cb14d5fbfdb9538a758c81a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4769a00a9cb14d5fbfdb9538a758c81a 2024-11-15T11:51:29,092 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 4769a00a9cb14d5fbfdb9538a758c81a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:29,092 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:29,092 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=12, startTime=1731671488652; duration=0sec 2024-11-15T11:51:29,092 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:29,092 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:29,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671549110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671549120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671549120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671549121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/fdab677889244778b3b5d151404ee2a7 2024-11-15T11:51:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/ed0df3c43ced4d4b8c67cd4907c0593a is 50, key is test_row_0/C:col10/1731671488433/Put/seqid=0 2024-11-15T11:51:29,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741896_1072 (size=12301) 2024-11-15T11:51:29,210 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/ed0df3c43ced4d4b8c67cd4907c0593a 2024-11-15T11:51:29,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/61c4e2221af44575badd10d0ca55fb9a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a 2024-11-15T11:51:29,222 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T11:51:29,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/fdab677889244778b3b5d151404ee2a7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7 2024-11-15T11:51:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T11:51:29,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/ed0df3c43ced4d4b8c67cd4907c0593a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a 2024-11-15T11:51:29,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a, entries=150, sequenceid=266, filesize=12.0 K 2024-11-15T11:51:29,238 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 18293b2c63865e9766ac3df4bea5a42d in 505ms, sequenceid=266, compaction requested=true 2024-11-15T11:51:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T11:51:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T11:51:29,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T11:51:29,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7450 sec 2024-11-15T11:51:29,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7540 sec 2024-11-15T11:51:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:29,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:29,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/38b9e03d17ee4b09ae3efcbcdf796c6a is 50, key is test_row_0/A:col10/1731671488784/Put/seqid=0 2024-11-15T11:51:29,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741897_1073 (size=14741) 2024-11-15T11:51:29,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/38b9e03d17ee4b09ae3efcbcdf796c6a 2024-11-15T11:51:29,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/460e1895a8434eff8e664e805d863587 is 50, key is test_row_0/B:col10/1731671488784/Put/seqid=0 2024-11-15T11:51:29,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671549388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741898_1074 (size=12301) 2024-11-15T11:51:29,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671549420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671549431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671549431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671549431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671549500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T11:51:29,640 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:29,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T11:51:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:29,643 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:29,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:29,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:29,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671549711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:29,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/460e1895a8434eff8e664e805d863587 2024-11-15T11:51:29,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/65884cfc1285494aa495838c9cbed633 is 50, key is test_row_0/C:col10/1731671488784/Put/seqid=0 2024-11-15T11:51:29,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741899_1075 (size=12301) 2024-11-15T11:51:29,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/65884cfc1285494aa495838c9cbed633 2024-11-15T11:51:29,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/38b9e03d17ee4b09ae3efcbcdf796c6a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a 2024-11-15T11:51:29,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a, entries=200, sequenceid=284, filesize=14.4 K 2024-11-15T11:51:29,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/460e1895a8434eff8e664e805d863587 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587 2024-11-15T11:51:29,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T11:51:29,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/65884cfc1285494aa495838c9cbed633 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633 2024-11-15T11:51:29,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T11:51:29,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 18293b2c63865e9766ac3df4bea5a42d in 564ms, sequenceid=284, compaction requested=true 2024-11-15T11:51:29,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:29,900 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:29,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:29,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:29,900 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:29,902 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51924 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:29,902 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:29,902 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,902 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/de5d934228844749985549031b9c8ff8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=50.7 K 2024-11-15T11:51:29,903 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49484 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:29,903 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:29,903 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,903 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c01ddda64caf4f30971210db0d03b0b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=48.3 K 2024-11-15T11:51:29,904 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5d934228844749985549031b9c8ff8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731671488181 2024-11-15T11:51:29,904 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c01ddda64caf4f30971210db0d03b0b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731671488181 2024-11-15T11:51:29,904 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting df3b4083ad924c3096d1a9ec165de08c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731671488272 2024-11-15T11:51:29,905 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9676c06cfc5c4736916e95263541c0f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731671488272 2024-11-15T11:51:29,905 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c4e2221af44575badd10d0ca55fb9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731671488417 2024-11-15T11:51:29,905 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fdab677889244778b3b5d151404ee2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731671488417 2024-11-15T11:51:29,906 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38b9e03d17ee4b09ae3efcbcdf796c6a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731671488784 2024-11-15T11:51:29,906 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 460e1895a8434eff8e664e805d863587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731671488784 2024-11-15T11:51:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:29,929 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:29,930 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/73fa6abe10854cd18f3b807aa90e3756 is 50, key is test_row_0/B:col10/1731671488784/Put/seqid=0 2024-11-15T11:51:29,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:29,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:29,941 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#64 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:29,942 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/19e80e24a0bc4ec6a830bbc62de05810 is 50, key is test_row_0/A:col10/1731671488784/Put/seqid=0 2024-11-15T11:51:29,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:29,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:29,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/7760414d05a245709deb8ce3edbd1c2c is 50, key is test_row_0/A:col10/1731671489383/Put/seqid=0 2024-11-15T11:51:29,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741901_1077 (size=13017) 2024-11-15T11:51:29,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741900_1076 (size=13017) 2024-11-15T11:51:29,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741902_1078 (size=14741) 2024-11-15T11:51:30,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/7760414d05a245709deb8ce3edbd1c2c 2024-11-15T11:51:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671549993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671549996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671549997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/411bdc4942364fa5896dcfa54e284cff is 50, key is test_row_0/B:col10/1731671489383/Put/seqid=0 2024-11-15T11:51:30,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671550005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671550020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741903_1079 (size=12301) 2024-11-15T11:51:30,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/411bdc4942364fa5896dcfa54e284cff 2024-11-15T11:51:30,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/566130cd0fd94e14ace89c51cf89a3bb is 50, key is test_row_0/C:col10/1731671489383/Put/seqid=0 2024-11-15T11:51:30,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741904_1080 (size=12301) 2024-11-15T11:51:30,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/566130cd0fd94e14ace89c51cf89a3bb 2024-11-15T11:51:30,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/7760414d05a245709deb8ce3edbd1c2c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c 2024-11-15T11:51:30,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c, entries=200, sequenceid=300, filesize=14.4 K 2024-11-15T11:51:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/411bdc4942364fa5896dcfa54e284cff as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff 2024-11-15T11:51:30,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff, entries=150, sequenceid=300, filesize=12.0 K 2024-11-15T11:51:30,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/566130cd0fd94e14ace89c51cf89a3bb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb 2024-11-15T11:51:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb, entries=150, sequenceid=300, filesize=12.0 K 2024-11-15T11:51:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671550111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 184ms, sequenceid=300, compaction requested=true 2024-11-15T11:51:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:30,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:30,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T11:51:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T11:51:30,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:30,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:30,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/4ffd392deb9145e18b9706420c2c0b06 is 50, key is test_row_0/A:col10/1731671490124/Put/seqid=0 2024-11-15T11:51:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671550152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671550154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671550160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741905_1081 (size=17181) 2024-11-15T11:51:30,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:30,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671550271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671550280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671550280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671550331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,384 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/19e80e24a0bc4ec6a830bbc62de05810 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/19e80e24a0bc4ec6a830bbc62de05810 2024-11-15T11:51:30,387 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/73fa6abe10854cd18f3b807aa90e3756 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73fa6abe10854cd18f3b807aa90e3756 2024-11-15T11:51:30,394 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into 19e80e24a0bc4ec6a830bbc62de05810(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:30,394 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:30,394 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=12, startTime=1731671489899; duration=0sec 2024-11-15T11:51:30,394 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T11:51:30,394 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:30,394 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:30,395 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:30,397 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:30,397 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:30,397 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,397 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4769a00a9cb14d5fbfdb9538a758c81a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=48.5 K 2024-11-15T11:51:30,398 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4769a00a9cb14d5fbfdb9538a758c81a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731671488272 2024-11-15T11:51:30,398 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0df3c43ced4d4b8c67cd4907c0593a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731671488417 2024-11-15T11:51:30,399 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65884cfc1285494aa495838c9cbed633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731671488784 2024-11-15T11:51:30,399 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566130cd0fd94e14ace89c51cf89a3bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731671489361 2024-11-15T11:51:30,406 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into 73fa6abe10854cd18f3b807aa90e3756(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:30,406 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:30,406 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=12, startTime=1731671489900; duration=0sec 2024-11-15T11:51:30,406 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:30,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:30,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:30,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:30,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:30,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:30,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,421 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:30,422 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/7d294268e3cd4530bd1ee9b1952cdf14 is 50, key is test_row_0/C:col10/1731671489383/Put/seqid=0 2024-11-15T11:51:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741906_1082 (size=13051) 2024-11-15T11:51:30,439 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/7d294268e3cd4530bd1ee9b1952cdf14 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7d294268e3cd4530bd1ee9b1952cdf14 2024-11-15T11:51:30,456 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 7d294268e3cd4530bd1ee9b1952cdf14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:30,456 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:30,457 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=12, startTime=1731671490125; duration=0sec 2024-11-15T11:51:30,457 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:30,457 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671550481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671550491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671550491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671550531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,534 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/4ffd392deb9145e18b9706420c2c0b06 2024-11-15T11:51:30,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0e570dd0d2154b9e9f315fb57b68ae4e is 50, key is test_row_0/B:col10/1731671490124/Put/seqid=0 2024-11-15T11:51:30,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741907_1083 (size=12301) 2024-11-15T11:51:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671550641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671550801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671550801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671550802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:30,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:30,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0e570dd0d2154b9e9f315fb57b68ae4e 2024-11-15T11:51:31,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/1679da9d0299443eb830ad7fa0def7c3 is 50, key is test_row_0/C:col10/1731671490124/Put/seqid=0 2024-11-15T11:51:31,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741908_1084 (size=12301) 2024-11-15T11:51:31,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/1679da9d0299443eb830ad7fa0def7c3 2024-11-15T11:51:31,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/4ffd392deb9145e18b9706420c2c0b06 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06 2024-11-15T11:51:31,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06, entries=250, sequenceid=319, filesize=16.8 K 2024-11-15T11:51:31,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/0e570dd0d2154b9e9f315fb57b68ae4e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e 2024-11-15T11:51:31,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T11:51:31,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/1679da9d0299443eb830ad7fa0def7c3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3 2024-11-15T11:51:31,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3, entries=150, sequenceid=319, filesize=12.0 K 2024-11-15T11:51:31,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 18293b2c63865e9766ac3df4bea5a42d in 1016ms, sequenceid=319, compaction requested=true 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:31,143 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:31,143 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:31,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:31,146 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:31,147 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:31,147 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,147 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73fa6abe10854cd18f3b807aa90e3756, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=36.7 K 2024-11-15T11:51:31,147 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:31,147 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:31,147 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,147 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/19e80e24a0bc4ec6a830bbc62de05810, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=43.9 K 2024-11-15T11:51:31,149 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e80e24a0bc4ec6a830bbc62de05810, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731671488784 2024-11-15T11:51:31,149 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fa6abe10854cd18f3b807aa90e3756, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731671488784 2024-11-15T11:51:31,150 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7760414d05a245709deb8ce3edbd1c2c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731671489361 2024-11-15T11:51:31,150 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 411bdc4942364fa5896dcfa54e284cff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731671489361 2024-11-15T11:51:31,150 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffd392deb9145e18b9706420c2c0b06, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731671489962 2024-11-15T11:51:31,150 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e570dd0d2154b9e9f315fb57b68ae4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731671489986 2024-11-15T11:51:31,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:31,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:31,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/23f97dd9a6cf4087b28ca08fb4ebd0af is 50, key is test_row_0/A:col10/1731671491152/Put/seqid=0 2024-11-15T11:51:31,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,197 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:31,197 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/d28420803d2b48189340b0dd71f44bc9 is 50, key is test_row_0/A:col10/1731671490124/Put/seqid=0 2024-11-15T11:51:31,206 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:31,207 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/4e7f8c9b3598491f9ba41c62531bda0e is 50, key is test_row_0/B:col10/1731671490124/Put/seqid=0 2024-11-15T11:51:31,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741909_1085 (size=14741) 2024-11-15T11:51:31,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/23f97dd9a6cf4087b28ca08fb4ebd0af 2024-11-15T11:51:31,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741910_1086 (size=13119) 2024-11-15T11:51:31,269 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/d28420803d2b48189340b0dd71f44bc9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/d28420803d2b48189340b0dd71f44bc9 2024-11-15T11:51:31,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741911_1087 (size=13119) 2024-11-15T11:51:31,291 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/4e7f8c9b3598491f9ba41c62531bda0e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4e7f8c9b3598491f9ba41c62531bda0e 2024-11-15T11:51:31,302 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into d28420803d2b48189340b0dd71f44bc9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:31,302 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:31,302 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=13, startTime=1731671491143; duration=0sec 2024-11-15T11:51:31,302 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into 4e7f8c9b3598491f9ba41c62531bda0e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:31,302 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:31,302 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=13, startTime=1731671491143; duration=0sec 2024-11-15T11:51:31,302 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:31,303 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:31,303 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:31,304 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:31,304 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:31,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/34bc639108944678b9c409c5b92f5d58 is 50, key is test_row_0/B:col10/1731671491152/Put/seqid=0 2024-11-15T11:51:31,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671551305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,308 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:31,308 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:31,308 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,308 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671551311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671551311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741912_1088 (size=12301) 2024-11-15T11:51:31,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671551321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671551410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671551541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,545 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,546 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671551621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:31,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:31,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/34bc639108944678b9c409c5b92f5d58 2024-11-15T11:51:31,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/efb62a6c129c411d885694e62d228f23 is 50, key is test_row_0/C:col10/1731671491152/Put/seqid=0 2024-11-15T11:51:31,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741913_1089 (size=12301) 2024-11-15T11:51:31,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/efb62a6c129c411d885694e62d228f23 2024-11-15T11:51:31,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/23f97dd9a6cf4087b28ca08fb4ebd0af as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af 2024-11-15T11:51:31,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af, entries=200, sequenceid=336, filesize=14.4 K 2024-11-15T11:51:31,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/34bc639108944678b9c409c5b92f5d58 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58 2024-11-15T11:51:31,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T11:51:31,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/efb62a6c129c411d885694e62d228f23 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23 2024-11-15T11:51:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:31,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T11:51:31,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 18293b2c63865e9766ac3df4bea5a42d in 652ms, sequenceid=336, compaction requested=true 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:31,809 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:31,809 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:31,813 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:31,813 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:31,813 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:31,813 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:31,814 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T11:51:31,815 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T11:51:31,816 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T11:51:31,816 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. because compaction request was cancelled 2024-11-15T11:51:31,816 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:31,818 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:31,818 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:31,818 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,818 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7d294268e3cd4530bd1ee9b1952cdf14, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=36.8 K 2024-11-15T11:51:31,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T11:51:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:31,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:31,820 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d294268e3cd4530bd1ee9b1952cdf14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731671489361 2024-11-15T11:51:31,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:31,821 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1679da9d0299443eb830ad7fa0def7c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731671489986 2024-11-15T11:51:31,822 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb62a6c129c411d885694e62d228f23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731671490150 2024-11-15T11:51:31,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eae865a002e74d10a6382624b2f52ad8 is 50, key is test_row_0/A:col10/1731671491275/Put/seqid=0 2024-11-15T11:51:31,843 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:31,843 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/7823194d5bbc4f449f844095527465a0 is 50, key is test_row_0/C:col10/1731671491152/Put/seqid=0 2024-11-15T11:51:31,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741915_1091 (size=12301) 2024-11-15T11:51:31,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eae865a002e74d10a6382624b2f52ad8 2024-11-15T11:51:31,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741914_1090 (size=13153) 2024-11-15T11:51:31,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/b8176e5a4b4248babd019fc931d473cc is 50, key is test_row_0/B:col10/1731671491275/Put/seqid=0 2024-11-15T11:51:31,897 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/7823194d5bbc4f449f844095527465a0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7823194d5bbc4f449f844095527465a0 2024-11-15T11:51:31,906 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 7823194d5bbc4f449f844095527465a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:31,906 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:31,906 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=13, startTime=1731671491809; duration=0sec 2024-11-15T11:51:31,907 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:31,907 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:31,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741916_1092 (size=12301) 2024-11-15T11:51:31,911 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/b8176e5a4b4248babd019fc931d473cc 2024-11-15T11:51:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/e6bedb2962404530945964cf63955941 is 50, key is test_row_0/C:col10/1731671491275/Put/seqid=0 2024-11-15T11:51:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:31,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741917_1093 (size=12301) 2024-11-15T11:51:31,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671551973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:31,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671552080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671552291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671552330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671552330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,333 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:32,333 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671552341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,344 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/e6bedb2962404530945964cf63955941 2024-11-15T11:51:32,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eae865a002e74d10a6382624b2f52ad8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8 2024-11-15T11:51:32,369 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T11:51:32,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/b8176e5a4b4248babd019fc931d473cc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc 2024-11-15T11:51:32,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T11:51:32,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/e6bedb2962404530945964cf63955941 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941 2024-11-15T11:51:32,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941, entries=150, sequenceid=356, filesize=12.0 K 2024-11-15T11:51:32,389 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 18293b2c63865e9766ac3df4bea5a42d in 570ms, sequenceid=356, compaction requested=true 2024-11-15T11:51:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T11:51:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T11:51:32,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T11:51:32,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7490 sec 2024-11-15T11:51:32,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.7560 sec 2024-11-15T11:51:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:32,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:32,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/42f4665fbde145749fc7e3ee85d2de53 is 50, key is test_row_0/A:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741918_1094 (size=12301) 2024-11-15T11:51:32,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/42f4665fbde145749fc7e3ee85d2de53 2024-11-15T11:51:32,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/e6c215f74c434b419257ea84bd79423c is 50, key is test_row_0/B:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:32,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671552673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741919_1095 (size=12301) 2024-11-15T11:51:32,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671552781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:32,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:32,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671552991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:32,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:33,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/e6c215f74c434b419257ea84bd79423c 2024-11-15T11:51:33,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/9a292185d36642a3add8261086e667b1 is 50, key is test_row_0/C:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:33,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741920_1096 (size=12301) 2024-11-15T11:51:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671553301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:33,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/9a292185d36642a3add8261086e667b1 2024-11-15T11:51:33,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/42f4665fbde145749fc7e3ee85d2de53 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53 2024-11-15T11:51:33,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T11:51:33,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/e6c215f74c434b419257ea84bd79423c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c 2024-11-15T11:51:33,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T11:51:33,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/9a292185d36642a3add8261086e667b1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1 2024-11-15T11:51:33,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T11:51:33,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18293b2c63865e9766ac3df4bea5a42d in 945ms, sequenceid=372, compaction requested=true 2024-11-15T11:51:33,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:33,547 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:33,549 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:33,549 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52462 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:33,549 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:33,549 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:33,550 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/d28420803d2b48189340b0dd71f44bc9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=51.2 K 2024-11-15T11:51:33,550 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d28420803d2b48189340b0dd71f44bc9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731671489986 2024-11-15T11:51:33,551 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f97dd9a6cf4087b28ca08fb4ebd0af, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731671490138 2024-11-15T11:51:33,552 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:33,552 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting eae865a002e74d10a6382624b2f52ad8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731671491257 2024-11-15T11:51:33,553 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:33,553 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:33,553 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4e7f8c9b3598491f9ba41c62531bda0e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=48.8 K 2024-11-15T11:51:33,553 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f4665fbde145749fc7e3ee85d2de53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:33,553 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7f8c9b3598491f9ba41c62531bda0e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731671489986 2024-11-15T11:51:33,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:51:33,554 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34bc639108944678b9c409c5b92f5d58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731671490150 2024-11-15T11:51:33,555 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8176e5a4b4248babd019fc931d473cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731671491257 2024-11-15T11:51:33,556 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c215f74c434b419257ea84bd79423c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:33,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:33,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:33,588 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:33,589 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/c4dfa1562ca84602898fd98ad763c64b is 50, key is test_row_0/B:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:33,600 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:33,600 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ed67eb6cec9c43fcb7c80b9b46b957ff is 50, key is test_row_0/A:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:33,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ec60b8aab43442aabd50ff07a0aa01ae is 50, key is test_row_0/A:col10/1731671492671/Put/seqid=0 2024-11-15T11:51:33,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741921_1097 (size=13255) 2024-11-15T11:51:33,646 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/c4dfa1562ca84602898fd98ad763c64b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c4dfa1562ca84602898fd98ad763c64b 2024-11-15T11:51:33,654 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into c4dfa1562ca84602898fd98ad763c64b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:33,654 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:33,654 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=12, startTime=1731671493548; duration=0sec 2024-11-15T11:51:33,655 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:33,655 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:33,655 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:33,656 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:33,657 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:33,657 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:33,657 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7823194d5bbc4f449f844095527465a0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=36.9 K 2024-11-15T11:51:33,658 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7823194d5bbc4f449f844095527465a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731671490150 2024-11-15T11:51:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671553651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:33,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741922_1098 (size=13255) 2024-11-15T11:51:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741923_1099 (size=14741) 2024-11-15T11:51:33,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ec60b8aab43442aabd50ff07a0aa01ae 2024-11-15T11:51:33,669 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6bedb2962404530945964cf63955941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731671491257 2024-11-15T11:51:33,671 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ed67eb6cec9c43fcb7c80b9b46b957ff as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed67eb6cec9c43fcb7c80b9b46b957ff 2024-11-15T11:51:33,672 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a292185d36642a3add8261086e667b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:33,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/a2d109c8165f4f64b26469a4e74ed3a2 is 50, key is test_row_0/B:col10/1731671492671/Put/seqid=0 2024-11-15T11:51:33,688 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into ed67eb6cec9c43fcb7c80b9b46b957ff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:33,688 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:33,688 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=12, startTime=1731671493547; duration=0sec 2024-11-15T11:51:33,688 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:33,688 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:33,693 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:33,694 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/158bc4c8111549db842395172dced3c6 is 50, key is test_row_0/C:col10/1731671491956/Put/seqid=0 2024-11-15T11:51:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741924_1100 (size=12301) 2024-11-15T11:51:33,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/a2d109c8165f4f64b26469a4e74ed3a2 2024-11-15T11:51:33,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741925_1101 (size=13255) 2024-11-15T11:51:33,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/096affa5eed34aceb9dd76fe51583587 is 50, key is test_row_0/C:col10/1731671492671/Put/seqid=0 2024-11-15T11:51:33,739 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/158bc4c8111549db842395172dced3c6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/158bc4c8111549db842395172dced3c6 2024-11-15T11:51:33,749 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into 158bc4c8111549db842395172dced3c6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:33,749 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:33,749 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=13, startTime=1731671493549; duration=0sec 2024-11-15T11:51:33,749 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:33,750 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:33,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741926_1102 (size=12301) 2024-11-15T11:51:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671553771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T11:51:33,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:33,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T11:51:33,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:33,818 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:33,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671553821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:33,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T11:51:33,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:33,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:33,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:33,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671553980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:33,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T11:51:34,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:34,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:34,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:34,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:34,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:34,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/096affa5eed34aceb9dd76fe51583587 2024-11-15T11:51:34,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/ec60b8aab43442aabd50ff07a0aa01ae as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae 2024-11-15T11:51:34,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae, entries=200, sequenceid=391, filesize=14.4 K 2024-11-15T11:51:34,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/a2d109c8165f4f64b26469a4e74ed3a2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2 2024-11-15T11:51:34,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2, entries=150, sequenceid=391, filesize=12.0 K 2024-11-15T11:51:34,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/096affa5eed34aceb9dd76fe51583587 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587 2024-11-15T11:51:34,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587, entries=150, sequenceid=391, filesize=12.0 K 2024-11-15T11:51:34,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 18293b2c63865e9766ac3df4bea5a42d in 612ms, sequenceid=391, compaction requested=false 2024-11-15T11:51:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:34,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T11:51:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:34,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:51:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:34,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/da2e814bbdf845469ae63de0d657ceea is 50, key is test_row_0/A:col10/1731671493636/Put/seqid=0 2024-11-15T11:51:34,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:34,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:34,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741927_1103 (size=12301) 2024-11-15T11:51:34,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671554360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671554361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671554361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,365 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671554370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,378 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:34,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:34,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671554470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671554470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671554680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671554680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/da2e814bbdf845469ae63de0d657ceea 2024-11-15T11:51:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/ca2bf7c51bd1489caae22c0598b58927 is 50, key is test_row_0/B:col10/1731671493636/Put/seqid=0 2024-11-15T11:51:34,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741928_1104 (size=12301) 2024-11-15T11:51:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671554830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,832 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671554991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671554991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:35,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/ca2bf7c51bd1489caae22c0598b58927 2024-11-15T11:51:35,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/a5062a63fcf2473c8525499b2fb913bc is 50, key is test_row_0/C:col10/1731671493636/Put/seqid=0 2024-11-15T11:51:35,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741929_1105 (size=12301) 2024-11-15T11:51:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671555500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:35,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671555501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:35,556 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/a5062a63fcf2473c8525499b2fb913bc 2024-11-15T11:51:35,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/da2e814bbdf845469ae63de0d657ceea as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea 2024-11-15T11:51:35,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea, entries=150, sequenceid=407, filesize=12.0 K 2024-11-15T11:51:35,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/ca2bf7c51bd1489caae22c0598b58927 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927 2024-11-15T11:51:35,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927, entries=150, sequenceid=407, filesize=12.0 K 2024-11-15T11:51:35,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/a5062a63fcf2473c8525499b2fb913bc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc 2024-11-15T11:51:35,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc, entries=150, sequenceid=407, filesize=12.0 K 2024-11-15T11:51:35,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 18293b2c63865e9766ac3df4bea5a42d in 1312ms, sequenceid=407, compaction requested=true 2024-11-15T11:51:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T11:51:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T11:51:35,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T11:51:35,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-11-15T11:51:35,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7880 sec 2024-11-15T11:51:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T11:51:35,961 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:35,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-15T11:51:35,963 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:35,965 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:35,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:36,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T11:51:36,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:36,117 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:36,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:36,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/24d7e3388ace4350955579a637d51b44 is 50, key is test_row_0/A:col10/1731671494336/Put/seqid=0 2024-11-15T11:51:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741930_1106 (size=12301) 2024-11-15T11:51:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:36,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:36,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/24d7e3388ace4350955579a637d51b44 2024-11-15T11:51:36,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/264728cb64fb4c2ab0ca108860e92df7 is 50, key is test_row_0/B:col10/1731671494336/Put/seqid=0 2024-11-15T11:51:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741931_1107 (size=12301) 2024-11-15T11:51:36,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671556563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671556565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671556671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671556671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1731671556851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,852 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671556880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671556880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:36,943 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/264728cb64fb4c2ab0ca108860e92df7 2024-11-15T11:51:36,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/b0ad5b881f9342de9991614a3fe9166f is 50, key is test_row_0/C:col10/1731671494336/Put/seqid=0 2024-11-15T11:51:36,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741932_1108 (size=12301) 2024-11-15T11:51:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:37,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671557190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671557190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:37,381 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/b0ad5b881f9342de9991614a3fe9166f 2024-11-15T11:51:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/24d7e3388ace4350955579a637d51b44 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44 2024-11-15T11:51:37,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T11:51:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/264728cb64fb4c2ab0ca108860e92df7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7 2024-11-15T11:51:37,400 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T11:51:37,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/b0ad5b881f9342de9991614a3fe9166f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f 2024-11-15T11:51:37,406 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T11:51:37,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 18293b2c63865e9766ac3df4bea5a42d in 1290ms, sequenceid=426, compaction requested=true 2024-11-15T11:51:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:37,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-15T11:51:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-15T11:51:37,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T11:51:37,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-15T11:51:37,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4530 sec 2024-11-15T11:51:37,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T11:51:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:37,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:37,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:37,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8243cd5b1568491bb48b4e1a7a32f490 is 50, key is test_row_0/A:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:37,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741933_1109 (size=14741) 2024-11-15T11:51:37,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671557758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:37,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671557759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:37,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671557870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:37,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671557881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671558080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671558090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T11:51:38,111 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:38,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-15T11:51:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:38,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:38,114 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:38,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:38,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8243cd5b1568491bb48b4e1a7a32f490 2024-11-15T11:51:38,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2d11276266434c9cab4adfee99287abc is 50, key is test_row_0/B:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:38,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741934_1110 (size=12301) 2024-11-15T11:51:38,195 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T11:51:38,195 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T11:51:38,195 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:38,195 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:38,195 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T11:51:38,195 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:38,197 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T11:51:38,197 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:38,200 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,200 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,200 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,200 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,201 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,201 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:38,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:38,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:38,201 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:38,201 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:38,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671558391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56346 deadline: 1731671558391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,392 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56304 deadline: 1731671558400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671558400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,401 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 18293b2c63865e9766ac3df4bea5a42d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:38,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:38,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2d11276266434c9cab4adfee99287abc 2024-11-15T11:51:38,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/00f1d56efb7147818a2661133080068d is 50, key is test_row_0/C:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:38,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741935_1111 (size=12301) 2024-11-15T11:51:38,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:38,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:38,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56276 deadline: 1731671558901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56278 deadline: 1731671558910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:38,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:51:38,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/00f1d56efb7147818a2661133080068d 2024-11-15T11:51:38,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/8243cd5b1568491bb48b4e1a7a32f490 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490 2024-11-15T11:51:38,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490, entries=200, sequenceid=441, filesize=14.4 K 2024-11-15T11:51:38,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/2d11276266434c9cab4adfee99287abc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc 2024-11-15T11:51:38,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc, entries=150, sequenceid=441, filesize=12.0 K 2024-11-15T11:51:38,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/00f1d56efb7147818a2661133080068d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d 2024-11-15T11:51:38,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d, entries=150, sequenceid=441, filesize=12.0 K 2024-11-15T11:51:38,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 18293b2c63865e9766ac3df4bea5a42d in 1275ms, sequenceid=441, compaction requested=true 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:38,978 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:38,978 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18293b2c63865e9766ac3df4bea5a42d:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67339 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/A is initiating minor compaction (all files) 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/B is initiating minor compaction (all files) 2024-11-15T11:51:38,980 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/A in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,980 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/B in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:38,980 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c4dfa1562ca84602898fd98ad763c64b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=61.0 K 2024-11-15T11:51:38,980 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed67eb6cec9c43fcb7c80b9b46b957ff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=65.8 K 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c4dfa1562ca84602898fd98ad763c64b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:38,980 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed67eb6cec9c43fcb7c80b9b46b957ff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d109c8165f4f64b26469a4e74ed3a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731671492669 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec60b8aab43442aabd50ff07a0aa01ae, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731671492669 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2bf7c51bd1489caae22c0598b58927, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731671493632 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2e814bbdf845469ae63de0d657ceea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731671493632 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 264728cb64fb4c2ab0ca108860e92df7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671494336 2024-11-15T11:51:38,981 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d7e3388ace4350955579a637d51b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671494336 2024-11-15T11:51:38,982 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d11276266434c9cab4adfee99287abc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731671496553 2024-11-15T11:51:38,982 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8243cd5b1568491bb48b4e1a7a32f490, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731671496553 2024-11-15T11:51:38,992 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:38,992 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/999e1a75aa124908a3960f950fa092bc is 50, key is test_row_0/A:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:38,993 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:38,993 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/fa806a4d9b324824ac5db910f3e291cd is 50, key is test_row_0/B:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:39,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741936_1112 (size=13425) 2024-11-15T11:51:39,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741937_1113 (size=13425) 2024-11-15T11:51:39,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:39,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:39,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/87d21faed03846e2b90db06961894606 is 50, key is test_row_0/A:col10/1731671497755/Put/seqid=0 2024-11-15T11:51:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741938_1114 (size=12301) 2024-11-15T11:51:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:39,407 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/999e1a75aa124908a3960f950fa092bc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/999e1a75aa124908a3960f950fa092bc 2024-11-15T11:51:39,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/fa806a4d9b324824ac5db910f3e291cd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fa806a4d9b324824ac5db910f3e291cd 2024-11-15T11:51:39,414 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/B of 18293b2c63865e9766ac3df4bea5a42d into fa806a4d9b324824ac5db910f3e291cd(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:39,414 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/A of 18293b2c63865e9766ac3df4bea5a42d into 999e1a75aa124908a3960f950fa092bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:39,414 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/B, priority=11, startTime=1731671498978; duration=0sec 2024-11-15T11:51:39,414 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/A, priority=11, startTime=1731671498978; duration=0sec 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:B 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:A 2024-11-15T11:51:39,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T11:51:39,415 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T11:51:39,416 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 18293b2c63865e9766ac3df4bea5a42d/C is initiating minor compaction (all files) 2024-11-15T11:51:39,416 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18293b2c63865e9766ac3df4bea5a42d/C in TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:39,416 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/158bc4c8111549db842395172dced3c6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp, totalSize=61.0 K 2024-11-15T11:51:39,416 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 158bc4c8111549db842395172dced3c6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731671491956 2024-11-15T11:51:39,416 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 096affa5eed34aceb9dd76fe51583587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731671492669 2024-11-15T11:51:39,417 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a5062a63fcf2473c8525499b2fb913bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731671493632 2024-11-15T11:51:39,417 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ad5b881f9342de9991614a3fe9166f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671494336 2024-11-15T11:51:39,417 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f1d56efb7147818a2661133080068d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1731671496553 2024-11-15T11:51:39,427 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18293b2c63865e9766ac3df4bea5a42d#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:39,427 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/f2d4c31926644aa0ada13a4e786b2e28 is 50, key is test_row_0/C:col10/1731671497701/Put/seqid=0 2024-11-15T11:51:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741939_1115 (size=13425) 2024-11-15T11:51:39,443 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/87d21faed03846e2b90db06961894606 2024-11-15T11:51:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/7e323f2dddd341cf8bf46feeeee0e1d5 is 50, key is test_row_0/B:col10/1731671497755/Put/seqid=0 2024-11-15T11:51:39,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741940_1116 (size=12301) 2024-11-15T11:51:39,572 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:51:39,838 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/f2d4c31926644aa0ada13a4e786b2e28 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/f2d4c31926644aa0ada13a4e786b2e28 2024-11-15T11:51:39,846 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18293b2c63865e9766ac3df4bea5a42d/C of 18293b2c63865e9766ac3df4bea5a42d into f2d4c31926644aa0ada13a4e786b2e28(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:39,846 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:39,846 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., storeName=18293b2c63865e9766ac3df4bea5a42d/C, priority=11, startTime=1731671498978; duration=0sec 2024-11-15T11:51:39,847 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:39,847 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18293b2c63865e9766ac3df4bea5a42d:C 2024-11-15T11:51:39,856 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/7e323f2dddd341cf8bf46feeeee0e1d5 2024-11-15T11:51:39,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/140c8fe3a2784b5e8d9e4632f26c81e8 is 50, key is test_row_0/C:col10/1731671497755/Put/seqid=0 2024-11-15T11:51:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741941_1117 (size=12301) 2024-11-15T11:51:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:39,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. as already flushing 2024-11-15T11:51:39,912 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T11:51:39,912 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:39,912 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:39,912 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:39,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:39,922 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T11:51:39,922 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:39,922 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:39,922 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:39,922 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:40,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:40,269 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/140c8fe3a2784b5e8d9e4632f26c81e8 2024-11-15T11:51:40,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/87d21faed03846e2b90db06961894606 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/87d21faed03846e2b90db06961894606 2024-11-15T11:51:40,279 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/87d21faed03846e2b90db06961894606, entries=150, sequenceid=459, filesize=12.0 K 2024-11-15T11:51:40,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/7e323f2dddd341cf8bf46feeeee0e1d5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/7e323f2dddd341cf8bf46feeeee0e1d5 2024-11-15T11:51:40,285 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/7e323f2dddd341cf8bf46feeeee0e1d5, entries=150, sequenceid=459, filesize=12.0 K 2024-11-15T11:51:40,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/140c8fe3a2784b5e8d9e4632f26c81e8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/140c8fe3a2784b5e8d9e4632f26c81e8 2024-11-15T11:51:40,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/140c8fe3a2784b5e8d9e4632f26c81e8, entries=150, sequenceid=459, filesize=12.0 K 2024-11-15T11:51:40,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 18293b2c63865e9766ac3df4bea5a42d in 1258ms, sequenceid=459, compaction requested=false 2024-11-15T11:51:40,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 18293b2c63865e9766ac3df4bea5a42d: 2024-11-15T11:51:40,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:40,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-15T11:51:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-15T11:51:40,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T11:51:40,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1790 sec 2024-11-15T11:51:40,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.1840 sec 2024-11-15T11:51:40,902 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T11:51:40,902 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:40,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:40,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:40,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:42,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T11:51:42,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:48,475 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T11:51:48,475 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:48,475 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,475 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:48,492 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T11:51:48,492 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:51:48,492 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,492 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:48,492 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2714 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2732 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:51:48,493 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1214 2024-11-15T11:51:48,494 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3634 rows 2024-11-15T11:51:48,494 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1217 2024-11-15T11:51:48,494 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3648 rows 2024-11-15T11:51:48,494 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:51:48,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:51:48,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:48,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:51:48,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:51:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:48,508 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671508508"}]},"ts":"1731671508508"} 2024-11-15T11:51:48,510 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:51:48,510 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:51:48,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:51:48,515 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, UNASSIGN}] 2024-11-15T11:51:48,516 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, UNASSIGN 2024-11-15T11:51:48,517 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=18293b2c63865e9766ac3df4bea5a42d, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:48,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, UNASSIGN because future has completed 2024-11-15T11:51:48,520 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:51:48,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:48,677 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:48,678 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:51:48,678 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 18293b2c63865e9766ac3df4bea5a42d, disabling compactions & flushes 2024-11-15T11:51:48,678 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:48,678 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. after waiting 0 ms 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:48,679 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 18293b2c63865e9766ac3df4bea5a42d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=A 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=B 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18293b2c63865e9766ac3df4bea5a42d, store=C 2024-11-15T11:51:48,679 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:48,685 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eb1139a3aaf84d9f879dd5bf3036669a is 50, key is test_row_0/A:col10/1731671508490/Put/seqid=0 2024-11-15T11:51:48,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741942_1118 (size=9857) 2024-11-15T11:51:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:49,092 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eb1139a3aaf84d9f879dd5bf3036669a 2024-11-15T11:51:49,103 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f18592668e184da388ce90a7fb9a21aa is 50, key is test_row_0/B:col10/1731671508490/Put/seqid=0 2024-11-15T11:51:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741943_1119 (size=9857) 2024-11-15T11:51:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:49,508 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f18592668e184da388ce90a7fb9a21aa 2024-11-15T11:51:49,521 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/c257e82dd3504f7f8de8168c23b1a03d is 50, key is test_row_0/C:col10/1731671508490/Put/seqid=0 2024-11-15T11:51:49,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741944_1120 (size=9857) 2024-11-15T11:51:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:49,926 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/c257e82dd3504f7f8de8168c23b1a03d 2024-11-15T11:51:49,934 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/A/eb1139a3aaf84d9f879dd5bf3036669a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eb1139a3aaf84d9f879dd5bf3036669a 2024-11-15T11:51:49,942 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eb1139a3aaf84d9f879dd5bf3036669a, entries=100, sequenceid=470, filesize=9.6 K 2024-11-15T11:51:49,943 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/B/f18592668e184da388ce90a7fb9a21aa as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f18592668e184da388ce90a7fb9a21aa 2024-11-15T11:51:49,948 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f18592668e184da388ce90a7fb9a21aa, entries=100, sequenceid=470, filesize=9.6 K 2024-11-15T11:51:49,949 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/.tmp/C/c257e82dd3504f7f8de8168c23b1a03d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/c257e82dd3504f7f8de8168c23b1a03d 2024-11-15T11:51:49,954 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/c257e82dd3504f7f8de8168c23b1a03d, entries=100, sequenceid=470, filesize=9.6 K 2024-11-15T11:51:49,955 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 18293b2c63865e9766ac3df4bea5a42d in 1276ms, sequenceid=470, compaction requested=true 2024-11-15T11:51:49,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/15fb9db34bf4446e97f07eb867c25028, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e0b2eafde3c24c17bfbc1524137841da, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e8c9ede5f35b459f9fdce8b37a454db4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/631f31d3d481481bbc65a2516b4c3683, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/de5d934228844749985549031b9c8ff8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/19e80e24a0bc4ec6a830bbc62de05810, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/d28420803d2b48189340b0dd71f44bc9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed67eb6cec9c43fcb7c80b9b46b957ff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490] to archive 2024-11-15T11:51:49,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:51:49,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed65d5a3a7244a8eab338e5971a92ba4 2024-11-15T11:51:49,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f9eea9157a94163926ff59dde7a58d4 2024-11-15T11:51:49,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/cbef026738ea49b3881b811a80d54747 2024-11-15T11:51:49,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/aa741c0586bb463e9866bfa77f5dad19 2024-11-15T11:51:49,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/15fb9db34bf4446e97f07eb867c25028 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/15fb9db34bf4446e97f07eb867c25028 2024-11-15T11:51:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/82dd094efb9b4a48978fade9046cdc85 2024-11-15T11:51:49,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8bcb92efece449c6a931180f6930b99a 2024-11-15T11:51:49,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e0b2eafde3c24c17bfbc1524137841da to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e0b2eafde3c24c17bfbc1524137841da 2024-11-15T11:51:49,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/0bff829d67c04f4ea5732da6fed7b740 2024-11-15T11:51:49,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e8c9ede5f35b459f9fdce8b37a454db4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/e8c9ede5f35b459f9fdce8b37a454db4 2024-11-15T11:51:49,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8f14013bacee40bfa451b2df471d9a01 2024-11-15T11:51:49,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/16515791cef5405ca4797ad21c7366ea 2024-11-15T11:51:49,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/f7bfc96411004d5b9314d651b9065352 2024-11-15T11:51:49,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/5701e353732d41d5b0a3dd8d44a8b77c 2024-11-15T11:51:49,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/631f31d3d481481bbc65a2516b4c3683 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/631f31d3d481481bbc65a2516b4c3683 2024-11-15T11:51:49,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/466a54fac09b43d093c4d8f376bcf4fe 2024-11-15T11:51:49,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/c9e72b4e5f544e61b81ec87d875aedbb 2024-11-15T11:51:49,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/de5d934228844749985549031b9c8ff8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/de5d934228844749985549031b9c8ff8 2024-11-15T11:51:49,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/df3b4083ad924c3096d1a9ec165de08c 2024-11-15T11:51:49,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/61c4e2221af44575badd10d0ca55fb9a 2024-11-15T11:51:49,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/38b9e03d17ee4b09ae3efcbcdf796c6a 2024-11-15T11:51:49,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/19e80e24a0bc4ec6a830bbc62de05810 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/19e80e24a0bc4ec6a830bbc62de05810 2024-11-15T11:51:49,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/7760414d05a245709deb8ce3edbd1c2c 2024-11-15T11:51:49,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/4ffd392deb9145e18b9706420c2c0b06 2024-11-15T11:51:49,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/d28420803d2b48189340b0dd71f44bc9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/d28420803d2b48189340b0dd71f44bc9 2024-11-15T11:51:49,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/23f97dd9a6cf4087b28ca08fb4ebd0af 2024-11-15T11:51:49,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eae865a002e74d10a6382624b2f52ad8 2024-11-15T11:51:50,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed67eb6cec9c43fcb7c80b9b46b957ff to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ed67eb6cec9c43fcb7c80b9b46b957ff 2024-11-15T11:51:50,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/42f4665fbde145749fc7e3ee85d2de53 2024-11-15T11:51:50,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/ec60b8aab43442aabd50ff07a0aa01ae 2024-11-15T11:51:50,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/da2e814bbdf845469ae63de0d657ceea 2024-11-15T11:51:50,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/24d7e3388ace4350955579a637d51b44 2024-11-15T11:51:50,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/8243cd5b1568491bb48b4e1a7a32f490 2024-11-15T11:51:50,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/092a5912c7d44f688c8ae8f578bc1979, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/345b5b5dc4c4446e9ddbedb2a97ea459, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/683a747f6ff3482a91a75104978e0239, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ab88a955dc5c4ed09f510d45b18cc891, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c01ddda64caf4f30971210db0d03b0b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73fa6abe10854cd18f3b807aa90e3756, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4e7f8c9b3598491f9ba41c62531bda0e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c4dfa1562ca84602898fd98ad763c64b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc] to archive 2024-11-15T11:51:50,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:51:50,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/288d2da5c7304e6d96c03327759a3d24 2024-11-15T11:51:50,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3721e6c7d1c14e74b783c5aaf846187f 2024-11-15T11:51:50,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c96d88df734461aa51f515e5ba914fc 2024-11-15T11:51:50,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/092a5912c7d44f688c8ae8f578bc1979 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/092a5912c7d44f688c8ae8f578bc1979 2024-11-15T11:51:50,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4103630ec6da4b88adf612a63e072749 2024-11-15T11:51:50,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f5dc11fb69d944f4a2fff8d235c7a0b5 2024-11-15T11:51:50,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/345b5b5dc4c4446e9ddbedb2a97ea459 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/345b5b5dc4c4446e9ddbedb2a97ea459 2024-11-15T11:51:50,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/273c085c206e4f67b99997a338355d1a 2024-11-15T11:51:50,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/1b14175c7089400cacc621f23126d82b 2024-11-15T11:51:50,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/683a747f6ff3482a91a75104978e0239 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/683a747f6ff3482a91a75104978e0239 2024-11-15T11:51:50,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/3b7d6e2a589f46bdad6592980cb00c9c 2024-11-15T11:51:50,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2688a9980c524d6594f273a142965d16 2024-11-15T11:51:50,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/6d3caebd60214d1181eaaaea56854019 2024-11-15T11:51:50,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ab88a955dc5c4ed09f510d45b18cc891 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ab88a955dc5c4ed09f510d45b18cc891 2024-11-15T11:51:50,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73058b7bd2614628a5989d6a6598cb8d 2024-11-15T11:51:50,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/64df7efd470f454b8e02b65f5788ec8d 2024-11-15T11:51:50,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c01ddda64caf4f30971210db0d03b0b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c01ddda64caf4f30971210db0d03b0b5 2024-11-15T11:51:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0c6209c9c83f409ea0ab72883722664d 2024-11-15T11:51:50,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/9676c06cfc5c4736916e95263541c0f2 2024-11-15T11:51:50,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fdab677889244778b3b5d151404ee2a7 2024-11-15T11:51:50,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73fa6abe10854cd18f3b807aa90e3756 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/73fa6abe10854cd18f3b807aa90e3756 2024-11-15T11:51:50,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/460e1895a8434eff8e664e805d863587 2024-11-15T11:51:50,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/411bdc4942364fa5896dcfa54e284cff 2024-11-15T11:51:50,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4e7f8c9b3598491f9ba41c62531bda0e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/4e7f8c9b3598491f9ba41c62531bda0e 2024-11-15T11:51:50,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/0e570dd0d2154b9e9f315fb57b68ae4e 2024-11-15T11:51:50,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/34bc639108944678b9c409c5b92f5d58 2024-11-15T11:51:50,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/b8176e5a4b4248babd019fc931d473cc 2024-11-15T11:51:50,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c4dfa1562ca84602898fd98ad763c64b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/c4dfa1562ca84602898fd98ad763c64b 2024-11-15T11:51:50,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/e6c215f74c434b419257ea84bd79423c 2024-11-15T11:51:50,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/a2d109c8165f4f64b26469a4e74ed3a2 2024-11-15T11:51:50,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/ca2bf7c51bd1489caae22c0598b58927 2024-11-15T11:51:50,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/264728cb64fb4c2ab0ca108860e92df7 2024-11-15T11:51:50,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/2d11276266434c9cab4adfee99287abc 2024-11-15T11:51:50,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/37449478fdf24fa1b94acf0dbfab7393, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d32ed07a9af84300b728c613b1064140, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b6ddd234566947dfb09d90f966c46861, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/68d8ccc1eec34551b716f08fd3c14ef5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4769a00a9cb14d5fbfdb9538a758c81a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7d294268e3cd4530bd1ee9b1952cdf14, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7823194d5bbc4f449f844095527465a0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/158bc4c8111549db842395172dced3c6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d] to archive 2024-11-15T11:51:50,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:51:50,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/2a6e23619e374de88ffeb98b5ad5b105 2024-11-15T11:51:50,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/713925d53fdd42818e85e455ce173b9f 2024-11-15T11:51:50,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/679218196eaa4ad3a580c9dc2053d25c 2024-11-15T11:51:50,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/37449478fdf24fa1b94acf0dbfab7393 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/37449478fdf24fa1b94acf0dbfab7393 2024-11-15T11:51:50,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/93e0b0dc217246e8b9e53672f3450e62 2024-11-15T11:51:50,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/8994dfcd774542c98cbec42013103b46 2024-11-15T11:51:50,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d32ed07a9af84300b728c613b1064140 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d32ed07a9af84300b728c613b1064140 2024-11-15T11:51:50,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/5e2144c28d7d495587de1ccf3e21c227 2024-11-15T11:51:50,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/58b504aa2a974e4083fd778faa45159b 2024-11-15T11:51:50,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b6ddd234566947dfb09d90f966c46861 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b6ddd234566947dfb09d90f966c46861 2024-11-15T11:51:50,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/6cbe17d4440b4ceaae2484719ef44b31 2024-11-15T11:51:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4cab028948164f1ba9ff5cb04810cf95 2024-11-15T11:51:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/80391f371af8416d88d2efcfc6df6be2 2024-11-15T11:51:50,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/68d8ccc1eec34551b716f08fd3c14ef5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/68d8ccc1eec34551b716f08fd3c14ef5 2024-11-15T11:51:50,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/591c892e38814fe5b4b74438e0679021 2024-11-15T11:51:50,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/3a69e0e9fc984d3d878a5bf3002e3213 2024-11-15T11:51:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/dd2ff62e1ffc4cd0adcab4c02cee293c 2024-11-15T11:51:50,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4769a00a9cb14d5fbfdb9538a758c81a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/4769a00a9cb14d5fbfdb9538a758c81a 2024-11-15T11:51:50,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/d86843f091a1468f80d4e18c0a949f8e 2024-11-15T11:51:50,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/ed0df3c43ced4d4b8c67cd4907c0593a 2024-11-15T11:51:50,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/65884cfc1285494aa495838c9cbed633 2024-11-15T11:51:50,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7d294268e3cd4530bd1ee9b1952cdf14 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7d294268e3cd4530bd1ee9b1952cdf14 2024-11-15T11:51:50,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/566130cd0fd94e14ace89c51cf89a3bb 2024-11-15T11:51:50,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/1679da9d0299443eb830ad7fa0def7c3 2024-11-15T11:51:50,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7823194d5bbc4f449f844095527465a0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/7823194d5bbc4f449f844095527465a0 2024-11-15T11:51:50,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/efb62a6c129c411d885694e62d228f23 2024-11-15T11:51:50,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/e6bedb2962404530945964cf63955941 2024-11-15T11:51:50,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/158bc4c8111549db842395172dced3c6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/158bc4c8111549db842395172dced3c6 2024-11-15T11:51:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/9a292185d36642a3add8261086e667b1 2024-11-15T11:51:50,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/096affa5eed34aceb9dd76fe51583587 2024-11-15T11:51:50,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/a5062a63fcf2473c8525499b2fb913bc 2024-11-15T11:51:50,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/b0ad5b881f9342de9991614a3fe9166f 2024-11-15T11:51:50,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/00f1d56efb7147818a2661133080068d 2024-11-15T11:51:50,108 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=1 2024-11-15T11:51:50,109 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. 2024-11-15T11:51:50,109 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 18293b2c63865e9766ac3df4bea5a42d: Waiting for close lock at 1731671508678Running coprocessor pre-close hooks at 1731671508678Disabling compacts and flushes for region at 1731671508678Disabling writes for close at 1731671508679 (+1 ms)Obtaining lock to block concurrent updates at 1731671508679Preparing flush snapshotting stores in 18293b2c63865e9766ac3df4bea5a42d at 1731671508679Finished memstore snapshotting TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731671508679Flushing stores of TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d. at 1731671508680 (+1 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/A: creating writer at 1731671508680Flushing 18293b2c63865e9766ac3df4bea5a42d/A: appending metadata at 1731671508685 (+5 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/A: closing flushed file at 1731671508685Flushing 18293b2c63865e9766ac3df4bea5a42d/B: creating writer at 1731671509098 (+413 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/B: appending metadata at 1731671509103 (+5 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/B: closing flushed file at 1731671509103Flushing 18293b2c63865e9766ac3df4bea5a42d/C: creating writer at 1731671509515 (+412 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/C: appending metadata at 1731671509520 (+5 ms)Flushing 18293b2c63865e9766ac3df4bea5a42d/C: closing flushed file at 1731671509520Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be902ae: reopening flushed file at 1731671509933 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2df4ac8d: reopening flushed file at 1731671509942 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb77add: reopening flushed file at 1731671509948 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 18293b2c63865e9766ac3df4bea5a42d in 1276ms, sequenceid=470, compaction requested=true at 1731671509955 (+7 ms)Writing region close event to WAL at 1731671510104 (+149 ms)Running coprocessor post-close hooks at 1731671510109 (+5 ms)Closed at 1731671510109 2024-11-15T11:51:50,112 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:50,113 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=18293b2c63865e9766ac3df4bea5a42d, regionState=CLOSED 2024-11-15T11:51:50,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:51:50,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T11:51:50,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 18293b2c63865e9766ac3df4bea5a42d, server=7adf9b3d9d04,44479,1731671471201 in 1.5960 sec 2024-11-15T11:51:50,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T11:51:50,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18293b2c63865e9766ac3df4bea5a42d, UNASSIGN in 1.6030 sec 2024-11-15T11:51:50,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-15T11:51:50,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6100 sec 2024-11-15T11:51:50,125 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671510124"}]},"ts":"1731671510124"} 2024-11-15T11:51:50,126 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:51:50,126 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:51:50,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6260 sec 2024-11-15T11:51:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T11:51:50,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:50,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:51:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,658 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T11:51:50,660 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,665 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:50,670 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/recovered.edits] 2024-11-15T11:51:50,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/87d21faed03846e2b90db06961894606 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/87d21faed03846e2b90db06961894606 2024-11-15T11:51:50,675 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/999e1a75aa124908a3960f950fa092bc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/999e1a75aa124908a3960f950fa092bc 2024-11-15T11:51:50,677 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eb1139a3aaf84d9f879dd5bf3036669a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/A/eb1139a3aaf84d9f879dd5bf3036669a 2024-11-15T11:51:50,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/7e323f2dddd341cf8bf46feeeee0e1d5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/7e323f2dddd341cf8bf46feeeee0e1d5 2024-11-15T11:51:50,680 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f18592668e184da388ce90a7fb9a21aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/f18592668e184da388ce90a7fb9a21aa 2024-11-15T11:51:50,681 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fa806a4d9b324824ac5db910f3e291cd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/B/fa806a4d9b324824ac5db910f3e291cd 2024-11-15T11:51:50,684 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/140c8fe3a2784b5e8d9e4632f26c81e8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/140c8fe3a2784b5e8d9e4632f26c81e8 2024-11-15T11:51:50,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/c257e82dd3504f7f8de8168c23b1a03d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/c257e82dd3504f7f8de8168c23b1a03d 2024-11-15T11:51:50,686 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/f2d4c31926644aa0ada13a4e786b2e28 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/C/f2d4c31926644aa0ada13a4e786b2e28 2024-11-15T11:51:50,689 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/recovered.edits/473.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d/recovered.edits/473.seqid 2024-11-15T11:51:50,689 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/18293b2c63865e9766ac3df4bea5a42d 2024-11-15T11:51:50,690 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:51:50,695 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T11:51:50,703 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:51:50,738 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:51:50,739 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,740 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:51:50,740 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671510740"}]},"ts":"9223372036854775807"} 2024-11-15T11:51:50,744 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:51:50,744 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 18293b2c63865e9766ac3df4bea5a42d, NAME => 'TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:51:50,744 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:51:50,745 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671510745"}]},"ts":"9223372036854775807"} 2024-11-15T11:51:50,748 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:51:50,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-15T11:51:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T11:51:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:51:50,772 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:50,786 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=241 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7adf9b3d9d04:44479-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-160368578_22 at /127.0.0.1:43136 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=324 (was 224) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10451 (was 10952) 2024-11-15T11:51:50,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=241, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=324, ProcessCount=11, AvailableMemoryMB=10451 2024-11-15T11:51:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:51:50,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:51:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:50,802 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:51:50,802 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:50,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-15T11:51:50,803 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:51:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741945_1121 (size=1061) 2024-11-15T11:51:50,837 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T11:51:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:51,211 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:51:51,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741946_1122 (size=53) 2024-11-15T11:51:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:51,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:51,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a3efb8ce7147f2125466d71154ac42aa, disabling compactions & flushes 2024-11-15T11:51:51,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. after waiting 0 ms 2024-11-15T11:51:51,618 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,619 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,619 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a3efb8ce7147f2125466d71154ac42aa: Waiting for close lock at 1731671511618Disabling compacts and flushes for region at 1731671511618Disabling writes for close at 1731671511618Writing region close event to WAL at 1731671511619 (+1 ms)Closed at 1731671511619 2024-11-15T11:51:51,620 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:51:51,621 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671511620"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671511620"}]},"ts":"1731671511620"} 2024-11-15T11:51:51,623 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:51:51,625 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:51:51,626 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671511625"}]},"ts":"1731671511625"} 2024-11-15T11:51:51,628 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:51:51,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, ASSIGN}] 2024-11-15T11:51:51,629 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, ASSIGN 2024-11-15T11:51:51,630 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:51:51,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:51,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, ASSIGN because future has completed 2024-11-15T11:51:51,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:51,943 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,943 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:51:51,943 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,944 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:51,944 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,944 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,946 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,947 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:51,949 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName A 2024-11-15T11:51:51,949 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:51,950 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:51,950 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,951 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:51,952 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName B 2024-11-15T11:51:51,952 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:51,952 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:51,953 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,954 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:51,954 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName C 2024-11-15T11:51:51,954 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:51,955 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:51,955 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,955 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,956 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,956 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,957 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,957 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,958 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:51:51,959 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,962 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:51:51,962 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened a3efb8ce7147f2125466d71154ac42aa; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74507571, jitterRate=0.11024932563304901}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:51:51,962 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:51,964 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for a3efb8ce7147f2125466d71154ac42aa: Running coprocessor pre-open hook at 1731671511944Writing region info on filesystem at 1731671511944Initializing all the Stores at 1731671511945 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671511945Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671511946 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671511946Cleaning up temporary data from old regions at 1731671511957 (+11 ms)Running coprocessor post-open hooks at 1731671511962 (+5 ms)Region opened successfully at 1731671511964 (+2 ms) 2024-11-15T11:51:51,965 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., pid=34, masterSystemTime=1731671511939 2024-11-15T11:51:51,966 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,966 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:51,967 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:51,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:51:51,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T11:51:51,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 in 185 msec 2024-11-15T11:51:51,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T11:51:51,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, ASSIGN in 344 msec 2024-11-15T11:51:51,976 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:51:51,976 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671511976"}]},"ts":"1731671511976"} 2024-11-15T11:51:51,978 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:51:51,978 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:51:51,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-15T11:51:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T11:51:52,952 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:52,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:52,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:52,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:52,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:52,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f1c931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:52,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:52,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:52,966 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:52,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:52,970 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:51:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:51:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-15T11:51:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:51:52,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:51:52,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:51:52,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:51:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T11:51:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741947_1123 (size=1097) 2024-11-15T11:51:53,401 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T11:51:53,401 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T11:51:53,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:51:53,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE}] 2024-11-15T11:51:53,417 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE 2024-11-15T11:51:53,417 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:53,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE because future has completed 2024-11-15T11:51:53,420 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:51:53,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:53,573 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,573 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:51:53,573 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing a3efb8ce7147f2125466d71154ac42aa, disabling compactions & flushes 2024-11-15T11:51:53,573 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,573 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,573 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. after waiting 0 ms 2024-11-15T11:51:53,573 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,577 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T11:51:53,578 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,578 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for a3efb8ce7147f2125466d71154ac42aa: Waiting for close lock at 1731671513573Running coprocessor pre-close hooks at 1731671513573Disabling compacts and flushes for region at 1731671513573Disabling writes for close at 1731671513573Writing region close event to WAL at 1731671513574 (+1 ms)Running coprocessor post-close hooks at 1731671513578 (+4 ms)Closed at 1731671513578 2024-11-15T11:51:53,578 WARN [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: a3efb8ce7147f2125466d71154ac42aa to self. 2024-11-15T11:51:53,580 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,581 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=CLOSED 2024-11-15T11:51:53,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:51:53,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-15T11:51:53,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 in 163 msec 2024-11-15T11:51:53,585 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE; state=CLOSED, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=true 2024-11-15T11:51:53,736 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:53,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE because future has completed 2024-11-15T11:51:53,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:51:53,902 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,902 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:51:53,903 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,903 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:51:53,904 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,904 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,907 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,908 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:53,912 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName A 2024-11-15T11:51:53,914 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:53,915 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:53,915 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,916 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:53,916 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName B 2024-11-15T11:51:53,916 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:53,917 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:53,917 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,917 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:51:53,918 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a3efb8ce7147f2125466d71154ac42aa columnFamilyName C 2024-11-15T11:51:53,918 DEBUG [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:53,918 INFO [StoreOpener-a3efb8ce7147f2125466d71154ac42aa-1 {}] regionserver.HStore(327): Store=a3efb8ce7147f2125466d71154ac42aa/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:51:53,918 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,918 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,919 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,920 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,922 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,922 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,922 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:51:53,923 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,924 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened a3efb8ce7147f2125466d71154ac42aa; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61412980, jitterRate=-0.08487528562545776}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:51:53,924 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:53,926 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for a3efb8ce7147f2125466d71154ac42aa: Running coprocessor pre-open hook at 1731671513904Writing region info on filesystem at 1731671513904Initializing all the Stores at 1731671513906 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671513906Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671513906Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671513906Cleaning up temporary data from old regions at 1731671513922 (+16 ms)Running coprocessor post-open hooks at 1731671513924 (+2 ms)Region opened successfully at 1731671513926 (+2 ms) 2024-11-15T11:51:53,927 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., pid=39, masterSystemTime=1731671513894 2024-11-15T11:51:53,929 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,929 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:53,930 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=OPEN, openSeqNum=5, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:53,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:51:53,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-15T11:51:53,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 in 193 msec 2024-11-15T11:51:53,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T11:51:53,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, REOPEN/MOVE in 518 msec 2024-11-15T11:51:53,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-15T11:51:53,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 530 msec 2024-11-15T11:51:53,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-15T11:51:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-15T11:51:53,946 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:53,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,948 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c58126, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d1d783, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff00749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab9bf1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,956 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc7898a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,957 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a57d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1294540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a724e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,963 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f155475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551bb50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,966 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f20eb6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31553f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bccbc99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,974 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,976 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68896d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:51:53,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:51:53,978 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:51:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:51:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:51:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f4debe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:51:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:51:53,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:51:53,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:51:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:51:53,983 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,983 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,984 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,984 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:53,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T11:51:53,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:53,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:53,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:53,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:53,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:53,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:53,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:53,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:53,991 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:53,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:53,998 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,998 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:54,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:54,002 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:54,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,011 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:54,016 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:51:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:54,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:51:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:51:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:54,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:54,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:51:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:54,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:51:54,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e144022b1dac4f1489be1d69bdc03c3d_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671514020/Put/seqid=0 2024-11-15T11:51:54,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731671514036, max timestamp=1731671514073Num uniques 859; 2024-11-15T11:51:54,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T11:51:54,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731671514058, max timestamp=1731671514073Num uniques 859; 2024-11-15T11:51:54,088 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T11:51:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:54,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731671514058, max timestamp=1731671514077Num uniques 859; 2024-11-15T11:51:54,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-15T11:51:54,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741948_1124 (size=12154) 2024-11-15T11:51:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671574112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671574114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671574116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671574118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671574120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671574232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671574232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671574232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671574233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671574233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:54,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671574441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671574441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671574441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671574452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671574452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:54,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e144022b1dac4f1489be1d69bdc03c3d_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e144022b1dac4f1489be1d69bdc03c3d_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:54,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/ff1c8b69988d44eab1ecdb1f32c426fe, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:54,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/ff1c8b69988d44eab1ecdb1f32c426fe is 175, key is test_row_0/A:col10/1731671514020/Put/seqid=0 2024-11-15T11:51:54,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741949_1125 (size=30955) 2024-11-15T11:51:54,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/ff1c8b69988d44eab1ecdb1f32c426fe 2024-11-15T11:51:54,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/dd3fa20da34243dbbc76f0e03866a350 is 50, key is test_row_0/B:col10/1731671514020/Put/seqid=0 2024-11-15T11:51:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741950_1126 (size=12001) 2024-11-15T11:51:54,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/dd3fa20da34243dbbc76f0e03866a350 2024-11-15T11:51:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:54,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/4350fb6a5cc042abaa458a7fae5e57c9 is 50, key is test_row_0/C:col10/1731671514020/Put/seqid=0 2024-11-15T11:51:54,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741951_1127 (size=12001) 2024-11-15T11:51:54,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/4350fb6a5cc042abaa458a7fae5e57c9 2024-11-15T11:51:54,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/ff1c8b69988d44eab1ecdb1f32c426fe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe 2024-11-15T11:51:54,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T11:51:54,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/dd3fa20da34243dbbc76f0e03866a350 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350 2024-11-15T11:51:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T11:51:54,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/4350fb6a5cc042abaa458a7fae5e57c9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9 2024-11-15T11:51:54,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T11:51:54,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.15 KB/134300 for a3efb8ce7147f2125466d71154ac42aa in 677ms, sequenceid=15, compaction requested=false 2024-11-15T11:51:54,699 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T11:51:54,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:54,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=137.86 KB heapSize=388.63 KB 2024-11-15T11:51:54,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:54,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:54,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115efe2227687f74a0e9068576acbb16c5c_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671514757/Put/seqid=0 2024-11-15T11:51:54,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741952_1128 (size=17034) 2024-11-15T11:51:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671574791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671574790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,848 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:54,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115efe2227687f74a0e9068576acbb16c5c_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115efe2227687f74a0e9068576acbb16c5c_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:54,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671574846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671574847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,856 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/baf6f198029b4c3497510c9b644e2483, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671574848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/baf6f198029b4c3497510c9b644e2483 is 175, key is test_row_0/A:col10/1731671514757/Put/seqid=0 2024-11-15T11:51:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741953_1129 (size=48139) 2024-11-15T11:51:54,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:54,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671574951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671574951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671574960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671574960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671574961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:54,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:54,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:55,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:55,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671575160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671575160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671575171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671575171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671575171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:55,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:55,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:55,271 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/baf6f198029b4c3497510c9b644e2483 2024-11-15T11:51:55,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6241e76ff27c426d8f6859a53aebe373 is 50, key is test_row_0/B:col10/1731671514757/Put/seqid=0 2024-11-15T11:51:55,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741954_1130 (size=12001) 2024-11-15T11:51:55,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6241e76ff27c426d8f6859a53aebe373 2024-11-15T11:51:55,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/731984b1bbf04005b6c428f6af619261 is 50, key is test_row_0/C:col10/1731671514757/Put/seqid=0 2024-11-15T11:51:55,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741955_1131 (size=12001) 2024-11-15T11:51:55,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/731984b1bbf04005b6c428f6af619261 2024-11-15T11:51:55,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/baf6f198029b4c3497510c9b644e2483 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483 2024-11-15T11:51:55,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483, entries=250, sequenceid=52, filesize=47.0 K 2024-11-15T11:51:55,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6241e76ff27c426d8f6859a53aebe373 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373 2024-11-15T11:51:55,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T11:51:55,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/731984b1bbf04005b6c428f6af619261 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261 2024-11-15T11:51:55,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T11:51:55,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~144.57 KB/148040, heapSize ~408.50 KB/418304, currentSize=53.67 KB/54960 for a3efb8ce7147f2125466d71154ac42aa in 618ms, sequenceid=52, compaction requested=false 2024-11-15T11:51:55,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:55,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,407 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df3394775b37439bb831c4c863ab0924_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671514781/Put/seqid=0 2024-11-15T11:51:55,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741956_1132 (size=12154) 2024-11-15T11:51:55,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:55,449 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df3394775b37439bb831c4c863ab0924_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df3394775b37439bb831c4c863ab0924_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a9904996d4b8460ab34d1cba92473910, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a9904996d4b8460ab34d1cba92473910 is 175, key is test_row_0/A:col10/1731671514781/Put/seqid=0 2024-11-15T11:51:55,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741957_1133 (size=30955) 2024-11-15T11:51:55,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a9904996d4b8460ab34d1cba92473910 2024-11-15T11:51:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:55,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:55,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/ec5b12662a874af48778ca5eb6d07505 is 50, key is test_row_0/B:col10/1731671514781/Put/seqid=0 2024-11-15T11:51:55,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741958_1134 (size=12001) 2024-11-15T11:51:55,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/ec5b12662a874af48778ca5eb6d07505 2024-11-15T11:51:55,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/db06450041ba4aa58c4f3a6c1e6b9df6 is 50, key is test_row_0/C:col10/1731671514781/Put/seqid=0 2024-11-15T11:51:55,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741959_1135 (size=12001) 2024-11-15T11:51:55,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671515502, max timestamp=1731671515526Num uniques 859; 2024-11-15T11:51:55,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:51:55,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671515502, max timestamp=1731671515527Num uniques 859; 2024-11-15T11:51:55,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:51:55,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671515502, max timestamp=1731671515526Num uniques 859; 2024-11-15T11:51:55,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:51:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671575562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671575562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671575564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,574 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671575565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671575567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,645 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:51:55,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671575681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671575681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671575681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671575691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671575691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671575901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671575902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671575902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671575902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671575902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:55,917 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/db06450041ba4aa58c4f3a6c1e6b9df6 2024-11-15T11:51:55,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a9904996d4b8460ab34d1cba92473910 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910 2024-11-15T11:51:55,936 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910, entries=150, sequenceid=63, filesize=30.2 K 2024-11-15T11:51:55,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/ec5b12662a874af48778ca5eb6d07505 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505 2024-11-15T11:51:55,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T11:51:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/db06450041ba4aa58c4f3a6c1e6b9df6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6 2024-11-15T11:51:55,963 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T11:51:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a3efb8ce7147f2125466d71154ac42aa in 558ms, sequenceid=63, compaction requested=true 2024-11-15T11:51:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T11:51:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T11:51:55,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T11:51:55,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9500 sec 2024-11-15T11:51:55,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9890 sec 2024-11-15T11:51:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T11:51:56,141 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:56,142 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T11:51:56,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:56,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:56,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:56,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:56,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:51:56,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:56,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b943047693b4a528de80cb9a2d2c83a_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741960_1136 (size=12154) 2024-11-15T11:51:56,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:56,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671576268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671576268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671576268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671576269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671576272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159b943047693b4a528de80cb9a2d2c83a_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b943047693b4a528de80cb9a2d2c83a_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:56,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/61ffb62805f748a09835017133bd42dc, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:56,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/61ffb62805f748a09835017133bd42dc is 175, key is test_row_0/A:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741961_1137 (size=30955) 2024-11-15T11:51:56,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T11:51:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671576391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671576391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671576391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671576391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671576392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T11:51:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:56,568 INFO [master/7adf9b3d9d04:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T11:51:56,568 INFO [master/7adf9b3d9d04:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T11:51:56,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T11:51:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671576600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671576600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671576601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671576601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671576601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:56,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/61ffb62805f748a09835017133bd42dc 2024-11-15T11:51:56,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/8262e85aee0340bd856ec375769cfbdc is 50, key is test_row_0/B:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741962_1138 (size=12001) 2024-11-15T11:51:56,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/8262e85aee0340bd856ec375769cfbdc 2024-11-15T11:51:56,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T11:51:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:56,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:56,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/421cda639a8049f78434764c7fb55faf is 50, key is test_row_0/C:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:56,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741963_1139 (size=12001) 2024-11-15T11:51:56,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/421cda639a8049f78434764c7fb55faf 2024-11-15T11:51:56,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/61ffb62805f748a09835017133bd42dc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc 2024-11-15T11:51:56,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc, entries=150, sequenceid=102, filesize=30.2 K 2024-11-15T11:51:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/8262e85aee0340bd856ec375769cfbdc as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc 2024-11-15T11:51:56,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc, entries=150, sequenceid=102, filesize=11.7 K 2024-11-15T11:51:56,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/421cda639a8049f78434764c7fb55faf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf 2024-11-15T11:51:56,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf, entries=150, sequenceid=102, filesize=11.7 K 2024-11-15T11:51:56,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for a3efb8ce7147f2125466d71154ac42aa in 628ms, sequenceid=102, compaction requested=true 2024-11-15T11:51:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:56,845 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:56,845 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:56,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:56,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:56,847 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:56,847 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:51:56,847 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,847 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:56,847 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:51:56,847 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=137.7 K 2024-11-15T11:51:56,847 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,847 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=46.9 K 2024-11-15T11:51:56,847 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,848 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc] 2024-11-15T11:51:56,849 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting dd3fa20da34243dbbc76f0e03866a350, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671514000 2024-11-15T11:51:56,849 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1c8b69988d44eab1ecdb1f32c426fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671514000 2024-11-15T11:51:56,850 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf6f198029b4c3497510c9b644e2483, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731671514091 2024-11-15T11:51:56,850 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6241e76ff27c426d8f6859a53aebe373, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731671514115 2024-11-15T11:51:56,850 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5b12662a874af48778ca5eb6d07505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731671514781 2024-11-15T11:51:56,851 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8262e85aee0340bd856ec375769cfbdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:56,851 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9904996d4b8460ab34d1cba92473910, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731671514781 2024-11-15T11:51:56,851 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ffb62805f748a09835017133bd42dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:56,878 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:56,879 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#120 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:56,880 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/fe260f5e6c1a4a6385810e0c3e9695c1 is 50, key is test_row_0/B:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ac9606e4885148d9a2ef7e64b2200fd7_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:56,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741964_1140 (size=12139) 2024-11-15T11:51:56,889 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ac9606e4885148d9a2ef7e64b2200fd7_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:56,889 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ac9606e4885148d9a2ef7e64b2200fd7_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:56,901 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/fe260f5e6c1a4a6385810e0c3e9695c1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe260f5e6c1a4a6385810e0c3e9695c1 2024-11-15T11:51:56,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T11:51:56,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:51:56,915 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into fe260f5e6c1a4a6385810e0c3e9695c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:56,915 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:56,915 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=12, startTime=1731671516845; duration=0sec 2024-11-15T11:51:56,915 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:56,915 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:56,915 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:56,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:56,923 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:51:56,924 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:51:56,924 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:56,924 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=46.9 K 2024-11-15T11:51:56,924 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4350fb6a5cc042abaa458a7fae5e57c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671514000 2024-11-15T11:51:56,925 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 731984b1bbf04005b6c428f6af619261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731671514115 2024-11-15T11:51:56,925 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting db06450041ba4aa58c4f3a6c1e6b9df6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731671514781 2024-11-15T11:51:56,925 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 421cda639a8049f78434764c7fb55faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:56,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741965_1141 (size=4469) 2024-11-15T11:51:56,932 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#121 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:56,932 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/c597ec5a4f004eecb4c723bbc65d54e3 is 175, key is test_row_0/A:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0737d8e4edc4f30bb2484c5e16e2200_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671516269/Put/seqid=0 2024-11-15T11:51:56,958 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:56,959 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/eb939d14f263478fb5c267e28fc616fe is 50, key is test_row_0/C:col10/1731671516216/Put/seqid=0 2024-11-15T11:51:56,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671516951, max timestamp=1731671516969Num uniques 859; 2024-11-15T11:51:56,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:51:56,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671516951, max timestamp=1731671516969Num uniques 859; 2024-11-15T11:51:56,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:51:56,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741966_1142 (size=31093) 2024-11-15T11:51:57,002 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/c597ec5a4f004eecb4c723bbc65d54e3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3 2024-11-15T11:51:57,011 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into c597ec5a4f004eecb4c723bbc65d54e3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:57,012 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:57,012 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=12, startTime=1731671516845; duration=0sec 2024-11-15T11:51:57,012 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:57,012 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:51:57,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671576999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741967_1143 (size=12154) 2024-11-15T11:51:57,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741968_1144 (size=12139) 2024-11-15T11:51:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671577002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671577003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671577005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671577005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,026 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/eb939d14f263478fb5c267e28fc616fe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/eb939d14f263478fb5c267e28fc616fe 2024-11-15T11:51:57,036 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into eb939d14f263478fb5c267e28fc616fe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:57,036 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:57,036 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=12, startTime=1731671516846; duration=0sec 2024-11-15T11:51:57,036 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:57,036 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:51:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671577120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671577120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671577120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671577120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671577131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:57,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671577331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671577331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671577331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671577340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671577340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:57,422 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0737d8e4edc4f30bb2484c5e16e2200_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0737d8e4edc4f30bb2484c5e16e2200_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:57,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0262ce919a3042708b69b4faa74b97ea, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:57,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0262ce919a3042708b69b4faa74b97ea is 175, key is test_row_0/A:col10/1731671516269/Put/seqid=0 2024-11-15T11:51:57,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741969_1145 (size=30955) 2024-11-15T11:51:57,445 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0262ce919a3042708b69b4faa74b97ea 2024-11-15T11:51:57,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/7e1b84522e564d0097c36ecf942e64f2 is 50, key is test_row_0/B:col10/1731671516269/Put/seqid=0 2024-11-15T11:51:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741970_1146 (size=12001) 2024-11-15T11:51:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671577641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671577641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671577641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671577650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671577650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:57,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/7e1b84522e564d0097c36ecf942e64f2 2024-11-15T11:51:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/13b2d7cb1ece44c2b06c0860f3e8b0fa is 50, key is test_row_0/C:col10/1731671516269/Put/seqid=0 2024-11-15T11:51:57,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741971_1147 (size=12001) 2024-11-15T11:51:57,875 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/13b2d7cb1ece44c2b06c0860f3e8b0fa 2024-11-15T11:51:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0262ce919a3042708b69b4faa74b97ea as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea 2024-11-15T11:51:57,883 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea, entries=150, sequenceid=111, filesize=30.2 K 2024-11-15T11:51:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T11:51:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/7e1b84522e564d0097c36ecf942e64f2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2 2024-11-15T11:51:57,892 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2, entries=150, sequenceid=111, filesize=11.7 K 2024-11-15T11:51:57,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/13b2d7cb1ece44c2b06c0860f3e8b0fa as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa 2024-11-15T11:51:57,898 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa, entries=150, sequenceid=111, filesize=11.7 K 2024-11-15T11:51:57,899 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for a3efb8ce7147f2125466d71154ac42aa in 984ms, sequenceid=111, compaction requested=false 2024-11-15T11:51:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T11:51:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T11:51:57,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T11:51:57,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7570 sec 2024-11-15T11:51:57,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7630 sec 2024-11-15T11:51:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:58,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=154.31 KB heapSize=443.47 KB 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:58,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:51:58,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111521992a4cb60b4b249df82a08df28faec_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671578162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671578168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741972_1148 (size=14794) 2024-11-15T11:51:58,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671578171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671578171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111521992a4cb60b4b249df82a08df28faec_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111521992a4cb60b4b249df82a08df28faec_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671578171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/3f63fca16ca94d6a9a43572fff883f91, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:58,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/3f63fca16ca94d6a9a43572fff883f91 is 175, key is test_row_0/A:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741973_1149 (size=39749) 2024-11-15T11:51:58,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=69.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/3f63fca16ca94d6a9a43572fff883f91 2024-11-15T11:51:58,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6da1b98d2ca3446b8ca58a5b3fea4a14 is 50, key is test_row_0/B:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741974_1150 (size=12151) 2024-11-15T11:51:58,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6da1b98d2ca3446b8ca58a5b3fea4a14 2024-11-15T11:51:58,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6acca095675e47389832615f98af44e0 is 50, key is test_row_0/C:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741975_1151 (size=12151) 2024-11-15T11:51:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671578280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T11:51:58,292 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:51:58,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:51:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T11:51:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:51:58,296 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:51:58,297 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:51:58,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:51:58,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671578291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671578291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671578300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:51:58,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T11:51:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671578491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671578510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671578510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671578510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T11:51:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:51:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:51:58,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6acca095675e47389832615f98af44e0 2024-11-15T11:51:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/3f63fca16ca94d6a9a43572fff883f91 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91 2024-11-15T11:51:58,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91, entries=200, sequenceid=147, filesize=38.8 K 2024-11-15T11:51:58,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6da1b98d2ca3446b8ca58a5b3fea4a14 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14 2024-11-15T11:51:58,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T11:51:58,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6acca095675e47389832615f98af44e0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0 2024-11-15T11:51:58,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0, entries=150, sequenceid=147, filesize=11.9 K 2024-11-15T11:51:58,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~443.42 KB/454064, currentSize=33.54 KB/34350 for a3efb8ce7147f2125466d71154ac42aa in 530ms, sequenceid=147, compaction requested=true 2024-11-15T11:51:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:58,685 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:58,686 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:58,686 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:51:58,686 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,686 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=99.4 K 2024-11-15T11:51:58,686 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,686 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91] 2024-11-15T11:51:58,687 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c597ec5a4f004eecb4c723bbc65d54e3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:58,687 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0262ce919a3042708b69b4faa74b97ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731671516221 2024-11-15T11:51:58,688 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f63fca16ca94d6a9a43572fff883f91, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517000 2024-11-15T11:51:58,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:51:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:58,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:51:58,690 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:58,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:51:58,691 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:58,691 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:51:58,692 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,692 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe260f5e6c1a4a6385810e0c3e9695c1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=35.4 K 2024-11-15T11:51:58,692 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fe260f5e6c1a4a6385810e0c3e9695c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:58,693 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1b84522e564d0097c36ecf942e64f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731671516221 2024-11-15T11:51:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:58,693 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da1b98d2ca3446b8ca58a5b3fea4a14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517002 2024-11-15T11:51:58,698 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:58,701 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411153a1ed71d05374b2d84f4b3515c354364_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:58,705 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411153a1ed71d05374b2d84f4b3515c354364_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:58,705 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a1ed71d05374b2d84f4b3515c354364_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:58,708 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:58,709 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c09c3a0ae3ef4890a0dabe4c33cc249e is 50, key is test_row_0/B:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741976_1152 (size=4469) 2024-11-15T11:51:58,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741977_1153 (size=12391) 2024-11-15T11:51:58,723 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c09c3a0ae3ef4890a0dabe4c33cc249e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c09c3a0ae3ef4890a0dabe4c33cc249e 2024-11-15T11:51:58,730 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into c09c3a0ae3ef4890a0dabe4c33cc249e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:58,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:58,730 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=13, startTime=1731671518690; duration=0sec 2024-11-15T11:51:58,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:51:58,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:51:58,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:51:58,732 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:51:58,732 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:51:58,732 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,732 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/eb939d14f263478fb5c267e28fc616fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=35.4 K 2024-11-15T11:51:58,732 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting eb939d14f263478fb5c267e28fc616fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731671516216 2024-11-15T11:51:58,733 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b2d7cb1ece44c2b06c0860f3e8b0fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731671516221 2024-11-15T11:51:58,736 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6acca095675e47389832615f98af44e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517002 2024-11-15T11:51:58,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:58,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:51:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:51:58,756 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:58,756 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/c2626e8622e947c09a2859c25153df94 is 50, key is test_row_0/C:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:58,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741978_1154 (size=12391) 2024-11-15T11:51:58,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f80144090424d1bb5ec85f243aaac0a_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671518169/Put/seqid=0 2024-11-15T11:51:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741979_1155 (size=12304) 2024-11-15T11:51:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:51:58,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731671518821, max timestamp=1731671518841Num uniques 859; 2024-11-15T11:51:58,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731671518821, max timestamp=1731671518841Num uniques 859; 2024-11-15T11:51:58,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T11:51:58,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731671518821, max timestamp=1731671518841Num uniques 859; 2024-11-15T11:51:58,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T11:51:58,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-15T11:51:58,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671578870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671578872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671578874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671578875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:51:58,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671578980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671578991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671578991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:58,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:58,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671578991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:58,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,117 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#129 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:51:59,118 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b3a7a46745bb4e5993e6d24cc7cc0278 is 175, key is test_row_0/A:col10/1731671517002/Put/seqid=0 2024-11-15T11:51:59,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741980_1156 (size=31345) 2024-11-15T11:51:59,128 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b3a7a46745bb4e5993e6d24cc7cc0278 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278 2024-11-15T11:51:59,136 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into b3a7a46745bb4e5993e6d24cc7cc0278(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:59,136 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:59,136 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=13, startTime=1731671518684; duration=0sec 2024-11-15T11:51:59,136 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:59,136 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:51:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:51:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671579191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,199 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/c2626e8622e947c09a2859c25153df94 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/c2626e8622e947c09a2859c25153df94 2024-11-15T11:51:59,199 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in a3efb8ce7147f2125466d71154ac42aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:51:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671579201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671579202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671579202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671579202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,207 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f80144090424d1bb5ec85f243aaac0a_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f80144090424d1bb5ec85f243aaac0a_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:51:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5964db4c151645c48667e0a65e4a1d29, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:51:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5964db4c151645c48667e0a65e4a1d29 is 175, key is test_row_0/A:col10/1731671518169/Put/seqid=0 2024-11-15T11:51:59,214 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into c2626e8622e947c09a2859c25153df94(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:51:59,214 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:59,214 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=13, startTime=1731671518691; duration=0sec 2024-11-15T11:51:59,214 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:51:59,214 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:51:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741981_1157 (size=31105) 2024-11-15T11:51:59,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5964db4c151645c48667e0a65e4a1d29 2024-11-15T11:51:59,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2d264f0fd5294849a2d998b32c94ce35 is 50, key is test_row_0/B:col10/1731671518169/Put/seqid=0 2024-11-15T11:51:59,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741982_1158 (size=12151) 2024-11-15T11:51:59,249 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2d264f0fd5294849a2d998b32c94ce35 2024-11-15T11:51:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/99c227e9005b4462890f67ec71ddb233 is 50, key is test_row_0/C:col10/1731671518169/Put/seqid=0 2024-11-15T11:51:59,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741983_1159 (size=12151) 2024-11-15T11:51:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:51:59,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671579511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:51:59,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:51:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:51:59,684 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/99c227e9005b4462890f67ec71ddb233 2024-11-15T11:51:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5964db4c151645c48667e0a65e4a1d29 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29 2024-11-15T11:51:59,695 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29, entries=150, sequenceid=156, filesize=30.4 K 2024-11-15T11:51:59,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2d264f0fd5294849a2d998b32c94ce35 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35 2024-11-15T11:51:59,700 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T11:51:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/99c227e9005b4462890f67ec71ddb233 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233 2024-11-15T11:51:59,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T11:51:59,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for a3efb8ce7147f2125466d71154ac42aa in 953ms, sequenceid=156, compaction requested=false 2024-11-15T11:51:59,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:51:59,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:51:59,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T11:51:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T11:51:59,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T11:51:59,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-15T11:51:59,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4200 sec 2024-11-15T11:52:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:00,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=161.02 KB heapSize=441.23 KB 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:00,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671580024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671580025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671580030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671580032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a406ba6d604c466fa2c0f257551ef431_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:00,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741984_1160 (size=17284) 2024-11-15T11:52:00,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671580140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671580141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671580141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671580150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671580351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671580351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671580351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671580360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T11:52:00,441 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:00,442 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T11:52:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:00,443 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:00,444 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:00,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:00,449 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:00,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a406ba6d604c466fa2c0f257551ef431_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a406ba6d604c466fa2c0f257551ef431_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:00,454 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/96e0bcae5ef74871bc0c6e7e58a0360a, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:00,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/96e0bcae5ef74871bc0c6e7e58a0360a is 175, key is test_row_0/A:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:00,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741985_1161 (size=48389) 2024-11-15T11:52:00,461 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/96e0bcae5ef74871bc0c6e7e58a0360a 2024-11-15T11:52:00,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/86d8f7db34814cbbb22ef71d5fb68bd6 is 50, key is test_row_0/B:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:00,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741986_1162 (size=12151) 2024-11-15T11:52:00,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/86d8f7db34814cbbb22ef71d5fb68bd6 2024-11-15T11:52:00,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/1aaade8c834b45828fd7979492372cb9 is 50, key is test_row_0/C:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741987_1163 (size=12151) 2024-11-15T11:52:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:00,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T11:52:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671580660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671580660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671580661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671580670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:00,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:00,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T11:52:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:00,837 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T11:52:00,837 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T11:52:00,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T11:52:00,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/1aaade8c834b45828fd7979492372cb9 2024-11-15T11:52:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:00,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/96e0bcae5ef74871bc0c6e7e58a0360a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a 2024-11-15T11:52:00,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a, entries=250, sequenceid=195, filesize=47.3 K 2024-11-15T11:52:00,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/86d8f7db34814cbbb22ef71d5fb68bd6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6 2024-11-15T11:52:00,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:52:00,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/1aaade8c834b45828fd7979492372cb9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9 2024-11-15T11:52:00,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:52:00,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~461.11 KB/472176, currentSize=20.13 KB/20610 for a3efb8ce7147f2125466d71154ac42aa in 909ms, sequenceid=195, compaction requested=true 2024-11-15T11:52:00,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:00,933 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:00,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:00,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:00,933 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:00,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:00,934 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:00,934 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:00,934 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,934 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=108.2 K 2024-11-15T11:52:00,935 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,935 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a] 2024-11-15T11:52:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:00,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:00,935 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:00,935 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3a7a46745bb4e5993e6d24cc7cc0278, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517002 2024-11-15T11:52:00,935 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:00,935 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:00,936 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c09c3a0ae3ef4890a0dabe4c33cc249e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=35.8 K 2024-11-15T11:52:00,936 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5964db4c151645c48667e0a65e4a1d29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731671518164 2024-11-15T11:52:00,936 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c09c3a0ae3ef4890a0dabe4c33cc249e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517002 2024-11-15T11:52:00,936 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d264f0fd5294849a2d998b32c94ce35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731671518164 2024-11-15T11:52:00,937 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e0bcae5ef74871bc0c6e7e58a0360a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518863 2024-11-15T11:52:00,939 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d8f7db34814cbbb22ef71d5fb68bd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518869 2024-11-15T11:52:00,955 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:00,958 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157943c111a47747e9b7805dcaa398798a_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:00,960 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:00,960 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/62cff8bb64c0485f99058dab712fb187 is 50, key is test_row_0/B:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:00,961 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157943c111a47747e9b7805dcaa398798a_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:00,961 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157943c111a47747e9b7805dcaa398798a_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741989_1165 (size=4469) 2024-11-15T11:52:01,009 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#138 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:01,010 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5053f7cc99c44fd89f44e60ac68ed2f6 is 175, key is test_row_0/A:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741988_1164 (size=12493) 2024-11-15T11:52:01,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741990_1166 (size=31447) 2024-11-15T11:52:01,048 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/5053f7cc99c44fd89f44e60ac68ed2f6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6 2024-11-15T11:52:01,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T11:52:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:01,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:01,057 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 5053f7cc99c44fd89f44e60ac68ed2f6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:01,057 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:01,057 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=13, startTime=1731671520932; duration=0sec 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:01,057 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:01,057 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:01,058 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:01,059 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:01,060 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:01,061 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:01,061 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/c2626e8622e947c09a2859c25153df94, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=35.8 K 2024-11-15T11:52:01,062 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2626e8622e947c09a2859c25153df94, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731671517002 2024-11-15T11:52:01,062 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99c227e9005b4462890f67ec71ddb233, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731671518164 2024-11-15T11:52:01,063 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aaade8c834b45828fd7979492372cb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518869 2024-11-15T11:52:01,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115073cc7f6bdc349208812fdb4edbc1ad5_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671520030/Put/seqid=0 2024-11-15T11:52:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:01,083 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:01,083 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/3a1bb6cd56054d6496760910378c1e8d is 50, key is test_row_0/C:col10/1731671520022/Put/seqid=0 2024-11-15T11:52:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741991_1167 (size=12304) 2024-11-15T11:52:01,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:01,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115073cc7f6bdc349208812fdb4edbc1ad5_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115073cc7f6bdc349208812fdb4edbc1ad5_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/e05bd528e53349f5a876348d3e9664cb, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/e05bd528e53349f5a876348d3e9664cb is 175, key is test_row_0/A:col10/1731671520030/Put/seqid=0 2024-11-15T11:52:01,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741993_1169 (size=31105) 2024-11-15T11:52:01,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741992_1168 (size=12493) 2024-11-15T11:52:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:01,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:01,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671521182, max timestamp=1731671521202Num uniques 859; 2024-11-15T11:52:01,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:01,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671521182, max timestamp=1731671521202Num uniques 859; 2024-11-15T11:52:01,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:01,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671521182, max timestamp=1731671521202Num uniques 859; 2024-11-15T11:52:01,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671521220, max timestamp=1731671521240Num uniques 1118; 2024-11-15T11:52:01,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:01,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671521220, max timestamp=1731671521240Num uniques 1118; 2024-11-15T11:52:01,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:01,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671521220, max timestamp=1731671521240Num uniques 1118; 2024-11-15T11:52:01,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:01,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731671521254, max timestamp=1731671521278Num uniques 1127; 2024-11-15T11:52:01,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-15T11:52:01,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731671521254, max timestamp=1731671521278Num uniques 1127; 2024-11-15T11:52:01,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-15T11:52:01,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731671521254, max timestamp=1731671521278Num uniques 1127; 2024-11-15T11:52:01,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-15T11:52:01,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=586, cellSize=26836, totalHeapSize=61184, min timestamp=1731671521295, max timestamp=1731671521371Num uniques 1236; 2024-11-15T11:52:01,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=26836, suffix heap size=156512,new segment heap size=61184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=586 2024-11-15T11:52:01,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=586, cellSize=26836, totalHeapSize=61184, min timestamp=1731671521295, max timestamp=1731671521371Num uniques 1236; 2024-11-15T11:52:01,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=26836, suffix heap size=156512,new segment heap size=61184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=586 2024-11-15T11:52:01,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=586, cellSize=26836, totalHeapSize=61184, min timestamp=1731671521295, max timestamp=1731671521371Num uniques 1236; 2024-11-15T11:52:01,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=56606, new segment data size=26836, suffix heap size=156512,new segment heap size=61184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=586 2024-11-15T11:52:01,419 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/62cff8bb64c0485f99058dab712fb187 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/62cff8bb64c0485f99058dab712fb187 2024-11-15T11:52:01,454 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into 62cff8bb64c0485f99058dab712fb187(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:01,454 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671581403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671581403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,456 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=13, startTime=1731671520933; duration=0sec 2024-11-15T11:52:01,456 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:01,456 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671581456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671581456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671581456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,509 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/e05bd528e53349f5a876348d3e9664cb 2024-11-15T11:52:01,520 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/3a1bb6cd56054d6496760910378c1e8d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3a1bb6cd56054d6496760910378c1e8d 2024-11-15T11:52:01,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/9b56b3501dec4e5c9765362599fa38c4 is 50, key is test_row_0/B:col10/1731671520030/Put/seqid=0 2024-11-15T11:52:01,529 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into 3a1bb6cd56054d6496760910378c1e8d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:01,529 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:01,530 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=13, startTime=1731671520935; duration=0sec 2024-11-15T11:52:01,530 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:01,530 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:01,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741994_1170 (size=12151) 2024-11-15T11:52:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671581560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671581560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:01,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671581581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671581581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671581581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671581771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671581771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671581790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671581790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:01,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671581790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:01,944 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/9b56b3501dec4e5c9765362599fa38c4 2024-11-15T11:52:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8e027e87d6054fd39a833c34e2e15f78 is 50, key is test_row_0/C:col10/1731671520030/Put/seqid=0 2024-11-15T11:52:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741995_1171 (size=12151) 2024-11-15T11:52:02,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671582080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671582080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671582101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671582101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671582101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8e027e87d6054fd39a833c34e2e15f78 2024-11-15T11:52:02,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/e05bd528e53349f5a876348d3e9664cb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb 2024-11-15T11:52:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb, entries=150, sequenceid=202, filesize=30.4 K 2024-11-15T11:52:02,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/9b56b3501dec4e5c9765362599fa38c4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4 2024-11-15T11:52:02,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T11:52:02,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8e027e87d6054fd39a833c34e2e15f78 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78 2024-11-15T11:52:02,378 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78, entries=150, sequenceid=202, filesize=11.9 K 2024-11-15T11:52:02,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=181.14 KB/185490 for a3efb8ce7147f2125466d71154ac42aa in 1323ms, sequenceid=202, compaction requested=false 2024-11-15T11:52:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T11:52:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T11:52:02,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T11:52:02,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9360 sec 2024-11-15T11:52:02,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9410 sec 2024-11-15T11:52:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T11:52:02,591 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:02,592 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T11:52:02,594 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:02,595 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:02,596 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:02,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=187.85 KB heapSize=504.33 KB 2024-11-15T11:52:02,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671521400, max timestamp=1731671522594Num uniques 1445; 2024-11-15T11:52:02,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=66180, new segment data size=11220, suffix heap size=178248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1445, new segment cells count=245 2024-11-15T11:52:02,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671521400, max timestamp=1731671522594Num uniques 1445; 2024-11-15T11:52:02,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=66180, new segment data size=11220, suffix heap size=178248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1445, new segment cells count=245 2024-11-15T11:52:02,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671521400, max timestamp=1731671522594Num uniques 1445; 2024-11-15T11:52:02,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=66180, new segment data size=11220, suffix heap size=178248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1445, new segment cells count=245 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:02,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158af28f2b90644ec6a36735dae44c4ed9_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:02,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741996_1172 (size=14994) 2024-11-15T11:52:02,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671522625, max timestamp=1731671522635Num uniques 859; 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671522625, max timestamp=1731671522635Num uniques 859; 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671522625, max timestamp=1731671522635Num uniques 859; 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:02,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:02,648 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158af28f2b90644ec6a36735dae44c4ed9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158af28f2b90644ec6a36735dae44c4ed9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:02,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/9b03468af6454f5eb54c254e62e839e2, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:02,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/9b03468af6454f5eb54c254e62e839e2 is 175, key is test_row_0/A:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:02,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741997_1173 (size=39949) 2024-11-15T11:52:02,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/9b03468af6454f5eb54c254e62e839e2 2024-11-15T11:52:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671582665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671582668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=518, cellSize=23718, totalHeapSize=54112, min timestamp=1731671522651, max timestamp=1731671522670Num uniques 1068; 2024-11-15T11:52:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671582670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=23718, suffix heap size=139040,new segment heap size=54112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=518 2024-11-15T11:52:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671522663, max timestamp=1731671522670Num uniques 1068; 2024-11-15T11:52:02,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T11:52:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671522663, max timestamp=1731671522670Num uniques 1068; 2024-11-15T11:52:02,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T11:52:02,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a699ba72897c41b9afa8f58fc0a8fd62 is 50, key is test_row_0/B:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:02,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741998_1174 (size=12301) 2024-11-15T11:52:02,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a699ba72897c41b9afa8f58fc0a8fd62 2024-11-15T11:52:02,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/acebee9bf65047eab025e832bb848bfa is 50, key is test_row_0/C:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:02,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741999_1175 (size=12301) 2024-11-15T11:52:02,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731671522696, max timestamp=1731671522729Num uniques 1377; 2024-11-15T11:52:02,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=63062, new segment data size=14972, suffix heap size=171176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1377, new segment cells count=327 2024-11-15T11:52:02,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731671522696, max timestamp=1731671522729Num uniques 1027; 2024-11-15T11:52:02,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-15T11:52:02,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731671522696, max timestamp=1731671522729Num uniques 1027; 2024-11-15T11:52:02,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-15T11:52:02,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:02,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:02,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671582808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731671522768, max timestamp=1731671522813Num uniques 1186; 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=17676, suffix heap size=151312,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=386 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731671522768, max timestamp=1731671522813Num uniques 1186; 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=17676, suffix heap size=151312,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=386 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=386, cellSize=17676, totalHeapSize=40384, min timestamp=1731671522768, max timestamp=1731671522813Num uniques 1186; 2024-11-15T11:52:02,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=17676, suffix heap size=151312,new segment heap size=40384 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=386 2024-11-15T11:52:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671582839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671582841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671582843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671582845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:02,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:02,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:02,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671582920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671582951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671582961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671582961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671582961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:02,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/acebee9bf65047eab025e832bb848bfa 2024-11-15T11:52:03,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/9b03468af6454f5eb54c254e62e839e2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2 2024-11-15T11:52:03,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2, entries=200, sequenceid=293, filesize=39.0 K 2024-11-15T11:52:03,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a699ba72897c41b9afa8f58fc0a8fd62 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62 2024-11-15T11:52:03,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671583141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T11:52:03,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/acebee9bf65047eab025e832bb848bfa as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa 2024-11-15T11:52:03,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T11:52:03,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~78.05 KB/79920, currentSize=161.02 KB/164880 for a3efb8ce7147f2125466d71154ac42aa in 566ms, sequenceid=293, compaction requested=true 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:03,163 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:03,163 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:03,165 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,165 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,165 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/62cff8bb64c0485f99058dab712fb187, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=36.1 K 2024-11-15T11:52:03,165 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=100.1 K 2024-11-15T11:52:03,165 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2] 2024-11-15T11:52:03,165 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 62cff8bb64c0485f99058dab712fb187, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518869 2024-11-15T11:52:03,166 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5053f7cc99c44fd89f44e60ac68ed2f6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518869 2024-11-15T11:52:03,166 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b56b3501dec4e5c9765362599fa38c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731671520023 2024-11-15T11:52:03,166 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05bd528e53349f5a876348d3e9664cb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731671520023 2024-11-15T11:52:03,166 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a699ba72897c41b9afa8f58fc0a8fd62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:03,166 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b03468af6454f5eb54c254e62e839e2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731671522833, max timestamp=1731671523163Num uniques 1245; 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731671522833, max timestamp=1731671523163Num uniques 1245; 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731671522833, max timestamp=1731671523163Num uniques 1245; 2024-11-15T11:52:03,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=57020, new segment data size=8930, suffix heap size=157448,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1245, new segment cells count=195 2024-11-15T11:52:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:03,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=53.67 KB heapSize=142.55 KB 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:03,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:03,181 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:03,181 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/243ac8f3447849b182ad44b1f829685f is 50, key is test_row_0/B:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:03,182 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:03,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111575ed5e49edbc4868b28ac7b71bed93d6_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671523174/Put/seqid=0 2024-11-15T11:52:03,196 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cd41235b6180418dbcd9f07bd578bac2_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:03,198 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cd41235b6180418dbcd9f07bd578bac2_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:03,198 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cd41235b6180418dbcd9f07bd578bac2_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:03,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742000_1176 (size=12745) 2024-11-15T11:52:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671583214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671583215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671583217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671523196, max timestamp=1731671523217Num uniques 859; 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671523196, max timestamp=1731671523217Num uniques 859; 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671523196, max timestamp=1731671523217Num uniques 859; 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742002_1178 (size=27698) 2024-11-15T11:52:03,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:03,238 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111575ed5e49edbc4868b28ac7b71bed93d6_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575ed5e49edbc4868b28ac7b71bed93d6_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:03,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4af60d9caf80481f86d54b843ed30e07, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:03,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4af60d9caf80481f86d54b843ed30e07 is 175, key is test_row_0/A:col10/1731671523174/Put/seqid=0 2024-11-15T11:52:03,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742001_1177 (size=83485) 2024-11-15T11:52:03,244 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4af60d9caf80481f86d54b843ed30e07 2024-11-15T11:52:03,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742003_1179 (size=4469) 2024-11-15T11:52:03,246 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#148 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:03,247 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/21c0ff5054144b168feaea7c20a51bbd is 175, key is test_row_0/A:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742004_1180 (size=31699) 2024-11-15T11:52:03,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6aaf74efd89a423590db8b2f3cc2bee9 is 50, key is test_row_0/B:col10/1731671523174/Put/seqid=0 2024-11-15T11:52:03,267 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/21c0ff5054144b168feaea7c20a51bbd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd 2024-11-15T11:52:03,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742005_1181 (size=12301) 2024-11-15T11:52:03,274 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 21c0ff5054144b168feaea7c20a51bbd(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:03,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:03,274 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=13, startTime=1731671523163; duration=0sec 2024-11-15T11:52:03,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:03,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:03,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:03,278 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:03,278 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:03,279 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,279 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3a1bb6cd56054d6496760910378c1e8d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=36.1 K 2024-11-15T11:52:03,280 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a1bb6cd56054d6496760910378c1e8d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671518869 2024-11-15T11:52:03,280 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e027e87d6054fd39a833c34e2e15f78, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731671520023 2024-11-15T11:52:03,280 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting acebee9bf65047eab025e832bb848bfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:03,292 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:03,293 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/e0f4a922e0364bc2a243e0357799114c is 50, key is test_row_0/C:col10/1731671522592/Put/seqid=0 2024-11-15T11:52:03,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742006_1182 (size=12745) 2024-11-15T11:52:03,330 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/e0f4a922e0364bc2a243e0357799114c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e0f4a922e0364bc2a243e0357799114c 2024-11-15T11:52:03,344 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into e0f4a922e0364bc2a243e0357799114c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:03,344 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:03,344 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=13, startTime=1731671523163; duration=0sec 2024-11-15T11:52:03,344 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:03,346 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:03,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671583336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671583337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671583341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671583341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671583451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671583451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671583451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671583461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671583461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,615 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/243ac8f3447849b182ad44b1f829685f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/243ac8f3447849b182ad44b1f829685f 2024-11-15T11:52:03,622 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into 243ac8f3447849b182ad44b1f829685f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:03,622 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:03,622 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=13, startTime=1731671523163; duration=0sec 2024-11-15T11:52:03,622 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:03,622 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:03,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671583670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671583670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671583671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671583671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6aaf74efd89a423590db8b2f3cc2bee9 2024-11-15T11:52:03,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/bbde7898869d4842a7414dae9073e16e is 50, key is test_row_0/C:col10/1731671523174/Put/seqid=0 2024-11-15T11:52:03,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742007_1183 (size=12301) 2024-11-15T11:52:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:03,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671583970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:03,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671583980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671583981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,981 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671583981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671583981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/bbde7898869d4842a7414dae9073e16e 2024-11-15T11:52:04,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4af60d9caf80481f86d54b843ed30e07 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07 2024-11-15T11:52:04,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07, entries=450, sequenceid=387, filesize=81.5 K 2024-11-15T11:52:04,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/6aaf74efd89a423590db8b2f3cc2bee9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9 2024-11-15T11:52:04,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9, entries=150, sequenceid=387, filesize=12.0 K 2024-11-15T11:52:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/bbde7898869d4842a7414dae9073e16e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e 2024-11-15T11:52:04,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-15T11:52:04,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~162.42 KB/166320, currentSize=127.47 KB/130530 for a3efb8ce7147f2125466d71154ac42aa in 940ms, sequenceid=387, compaction requested=false 2024-11-15T11:52:04,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:04,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T11:52:04,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:04,131 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:04,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590cfd550d5d24eb9a159a2fe481edf8c_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671523339/Put/seqid=0 2024-11-15T11:52:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742008_1184 (size=12454) 2024-11-15T11:52:04,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:04,157 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590cfd550d5d24eb9a159a2fe481edf8c_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590cfd550d5d24eb9a159a2fe481edf8c_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:04,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/1c24851f52154572a88d96ba72da3ea0, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:04,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/1c24851f52154572a88d96ba72da3ea0 is 175, key is test_row_0/A:col10/1731671523339/Put/seqid=0 2024-11-15T11:52:04,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742009_1185 (size=31255) 2024-11-15T11:52:04,168 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/1c24851f52154572a88d96ba72da3ea0 2024-11-15T11:52:04,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/be3c5bd65e404ed69f7a1dd71e622679 is 50, key is test_row_0/B:col10/1731671523339/Put/seqid=0 2024-11-15T11:52:04,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742010_1186 (size=12301) 2024-11-15T11:52:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:04,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671584500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671584501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671584503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671584506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,593 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/be3c5bd65e404ed69f7a1dd71e622679 2024-11-15T11:52:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6e1dd049289c4ddaa7fdf1713aa63ab1 is 50, key is test_row_0/C:col10/1731671523339/Put/seqid=0 2024-11-15T11:52:04,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742011_1187 (size=12301) 2024-11-15T11:52:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671584620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671584621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671584621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671584830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671584830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671584831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671584991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:04,997 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in a3efb8ce7147f2125466d71154ac42aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,010 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6e1dd049289c4ddaa7fdf1713aa63ab1 2024-11-15T11:52:05,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/1c24851f52154572a88d96ba72da3ea0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0 2024-11-15T11:52:05,027 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0, entries=150, sequenceid=426, filesize=30.5 K 2024-11-15T11:52:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/be3c5bd65e404ed69f7a1dd71e622679 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679 2024-11-15T11:52:05,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T11:52:05,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/6e1dd049289c4ddaa7fdf1713aa63ab1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1 2024-11-15T11:52:05,044 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1, entries=150, sequenceid=426, filesize=12.0 K 2024-11-15T11:52:05,045 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for a3efb8ce7147f2125466d71154ac42aa in 914ms, sequenceid=426, compaction requested=true 2024-11-15T11:52:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:05,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T11:52:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T11:52:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T11:52:05,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4550 sec 2024-11-15T11:52:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4630 sec 2024-11-15T11:52:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:05,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115451994cf2e754e509e469779f59b5b45_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742012_1188 (size=12454) 2024-11-15T11:52:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671585332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671585334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671585334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671585441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671585440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671585451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671585511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,517 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in a3efb8ce7147f2125466d71154ac42aa of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671585651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671585651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,665 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:05,672 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115451994cf2e754e509e469779f59b5b45_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115451994cf2e754e509e469779f59b5b45_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:05,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/787409e4cee449d2822950ba09c3610d, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:05,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/787409e4cee449d2822950ba09c3610d is 175, key is test_row_0/A:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:05,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671585671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742013_1189 (size=31251) 2024-11-15T11:52:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671585971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671585971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:05,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671585990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:05,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:06,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/787409e4cee449d2822950ba09c3610d 2024-11-15T11:52:06,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e078573f05904968986e55512479ba02 is 50, key is test_row_0/B:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:06,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742014_1190 (size=9857) 2024-11-15T11:52:06,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e078573f05904968986e55512479ba02 2024-11-15T11:52:06,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/32d0288f71cf4be4b0ee62d984736466 is 50, key is test_row_0/C:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:06,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742015_1191 (size=9857) 2024-11-15T11:52:06,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/32d0288f71cf4be4b0ee62d984736466 2024-11-15T11:52:06,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/787409e4cee449d2822950ba09c3610d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d 2024-11-15T11:52:06,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d, entries=150, sequenceid=438, filesize=30.5 K 2024-11-15T11:52:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e078573f05904968986e55512479ba02 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02 2024-11-15T11:52:06,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02, entries=100, sequenceid=438, filesize=9.6 K 2024-11-15T11:52:06,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/32d0288f71cf4be4b0ee62d984736466 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466 2024-11-15T11:52:06,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466, entries=100, sequenceid=438, filesize=9.6 K 2024-11-15T11:52:06,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a3efb8ce7147f2125466d71154ac42aa in 1104ms, sequenceid=438, compaction requested=true 2024-11-15T11:52:06,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:06,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:06,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:06,287 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:06,288 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:06,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:06,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:06,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:06,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:06,291 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 177690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:06,291 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:06,291 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,291 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=173.5 K 2024-11-15T11:52:06,291 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,291 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d] 2024-11-15T11:52:06,292 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:06,292 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:06,292 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,292 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/243ac8f3447849b182ad44b1f829685f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=46.1 K 2024-11-15T11:52:06,292 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 243ac8f3447849b182ad44b1f829685f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:06,293 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c0ff5054144b168feaea7c20a51bbd, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:06,293 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6aaf74efd89a423590db8b2f3cc2bee9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731671523171 2024-11-15T11:52:06,293 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4af60d9caf80481f86d54b843ed30e07, keycount=450, bloomtype=ROW, size=81.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731671522833 2024-11-15T11:52:06,294 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting be3c5bd65e404ed69f7a1dd71e622679, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671523323 2024-11-15T11:52:06,294 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c24851f52154572a88d96ba72da3ea0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671523323 2024-11-15T11:52:06,295 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e078573f05904968986e55512479ba02, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671524504 2024-11-15T11:52:06,295 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787409e4cee449d2822950ba09c3610d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671524500 2024-11-15T11:52:06,326 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:06,327 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/0c830f61d8b14b31bfeeb659d9f7fbab is 50, key is test_row_0/B:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:06,328 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,340 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150f1f7061133948449a091edd21625213_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,357 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150f1f7061133948449a091edd21625213_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,358 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150f1f7061133948449a091edd21625213_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742016_1192 (size=12881) 2024-11-15T11:52:06,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742017_1193 (size=4469) 2024-11-15T11:52:06,415 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#160 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:06,416 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/902bdca6b18240c0b34d776bb627df7c is 175, key is test_row_0/A:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742018_1194 (size=31942) 2024-11-15T11:52:06,444 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/902bdca6b18240c0b34d776bb627df7c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c 2024-11-15T11:52:06,453 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 902bdca6b18240c0b34d776bb627df7c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:06,453 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:06,453 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=12, startTime=1731671526287; duration=0sec 2024-11-15T11:52:06,453 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:06,453 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:06,453 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:06,455 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47204 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:06,455 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:06,455 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,455 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e0f4a922e0364bc2a243e0357799114c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=46.1 K 2024-11-15T11:52:06,460 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f4a922e0364bc2a243e0357799114c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731671521400 2024-11-15T11:52:06,461 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbde7898869d4842a7414dae9073e16e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731671523171 2024-11-15T11:52:06,462 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1dd049289c4ddaa7fdf1713aa63ab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731671523323 2024-11-15T11:52:06,464 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32d0288f71cf4be4b0ee62d984736466, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671524504 2024-11-15T11:52:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671525309, max timestamp=1731671526483Num uniques 859; 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671525309, max timestamp=1731671526483Num uniques 859; 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671525309, max timestamp=1731671526483Num uniques 859; 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:06,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:06,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:06,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,505 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:06,505 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/083efb1262af451781f2482daf5ad7e2 is 50, key is test_row_0/C:col10/1731671525160/Put/seqid=0 2024-11-15T11:52:06,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a5106552c55c4c61a1b25ccd49a81ee6_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671525331/Put/seqid=0 2024-11-15T11:52:06,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742019_1195 (size=12881) 2024-11-15T11:52:06,559 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/083efb1262af451781f2482daf5ad7e2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/083efb1262af451781f2482daf5ad7e2 2024-11-15T11:52:06,569 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into 083efb1262af451781f2482daf5ad7e2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:06,569 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:06,569 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=12, startTime=1731671526288; duration=0sec 2024-11-15T11:52:06,570 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:06,570 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671526537, max timestamp=1731671526574Num uniques 859; 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671526537, max timestamp=1731671526574Num uniques 859; 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671526537, max timestamp=1731671526574Num uniques 859; 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:06,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:06,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742020_1196 (size=14994) 2024-11-15T11:52:06,589 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:06,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a5106552c55c4c61a1b25ccd49a81ee6_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a5106552c55c4c61a1b25ccd49a81ee6_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:06,613 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/653420dc2c544e6e959a3642b988815d, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/653420dc2c544e6e959a3642b988815d is 175, key is test_row_0/A:col10/1731671525331/Put/seqid=0 2024-11-15T11:52:06,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742021_1197 (size=39949) 2024-11-15T11:52:06,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=460, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/653420dc2c544e6e959a3642b988815d 2024-11-15T11:52:06,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671526631, max timestamp=1731671526654Num uniques 1118; 2024-11-15T11:52:06,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671526631, max timestamp=1731671526654Num uniques 1118; 2024-11-15T11:52:06,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:06,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:06,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671526631, max timestamp=1731671526654Num uniques 1118; 2024-11-15T11:52:06,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=12268, suffix heap size=144240,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=268 2024-11-15T11:52:06,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/814dceced81445bfb0b455ab36e1fd65 is 50, key is test_row_0/B:col10/1731671525331/Put/seqid=0 2024-11-15T11:52:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742022_1198 (size=12301) 2024-11-15T11:52:06,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/814dceced81445bfb0b455ab36e1fd65 2024-11-15T11:52:06,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8afe4e49025143179b1987dadeaef39c is 50, key is test_row_0/C:col10/1731671525331/Put/seqid=0 2024-11-15T11:52:06,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742023_1199 (size=12301) 2024-11-15T11:52:06,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8afe4e49025143179b1987dadeaef39c 2024-11-15T11:52:06,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/653420dc2c544e6e959a3642b988815d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d 2024-11-15T11:52:06,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d, entries=200, sequenceid=460, filesize=39.0 K 2024-11-15T11:52:06,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/814dceced81445bfb0b455ab36e1fd65 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65 2024-11-15T11:52:06,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65, entries=150, sequenceid=460, filesize=12.0 K 2024-11-15T11:52:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671586760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:06,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671526685, max timestamp=1731671526760Num uniques 1127; 2024-11-15T11:52:06,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-15T11:52:06,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671526685, max timestamp=1731671526760Num uniques 1127; 2024-11-15T11:52:06,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-15T11:52:06,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8afe4e49025143179b1987dadeaef39c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c 2024-11-15T11:52:06,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671526685, max timestamp=1731671526760Num uniques 1127; 2024-11-15T11:52:06,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=10392, suffix heap size=145176,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=227 2024-11-15T11:52:06,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T11:52:06,771 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:06,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T11:52:06,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c, entries=150, sequenceid=460, filesize=12.0 K 2024-11-15T11:52:06,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:06,774 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:06,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:06,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=40.25 KB/41220 for a3efb8ce7147f2125466d71154ac42aa in 289ms, sequenceid=460, compaction requested=false 2024-11-15T11:52:06,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:06,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=53.67 KB heapSize=139.55 KB 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:06,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:06,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d5e1e2e809d64ec1a16e4234a7f9a575_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:06,809 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/0c830f61d8b14b31bfeeb659d9f7fbab as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/0c830f61d8b14b31bfeeb659d9f7fbab 2024-11-15T11:52:06,815 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into 0c830f61d8b14b31bfeeb659d9f7fbab(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:06,815 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:06,815 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=12, startTime=1731671526288; duration=0sec 2024-11-15T11:52:06,815 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:06,815 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:06,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742024_1200 (size=17534) 2024-11-15T11:52:06,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:06,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d5e1e2e809d64ec1a16e4234a7f9a575_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d5e1e2e809d64ec1a16e4234a7f9a575_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:06,845 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a0df92ddded840849c212ca8aa17fb9c, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:06,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a0df92ddded840849c212ca8aa17fb9c is 175, key is test_row_0/A:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:06,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671526819, max timestamp=1731671526852Num uniques 859; 2024-11-15T11:52:06,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:06,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671526819, max timestamp=1731671526852Num uniques 859; 2024-11-15T11:52:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671526819, max timestamp=1731671526852Num uniques 859; 2024-11-15T11:52:06,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:06,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:06,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742025_1201 (size=48639) 2024-11-15T11:52:06,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=519, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a0df92ddded840849c212ca8aa17fb9c 2024-11-15T11:52:06,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:06,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a5b35d7fb76649f186fab620c8626689 is 50, key is test_row_0/B:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:06,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:06,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671586913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:06,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671586917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:06,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:06,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T11:52:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:06,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:06,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671586919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:06,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:06,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:06,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742026_1202 (size=12301) 2024-11-15T11:52:06,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a5b35d7fb76649f186fab620c8626689 2024-11-15T11:52:06,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/e77a6fab8840486ca5f31ea7e5fd49b9 is 50, key is test_row_0/C:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:06,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742027_1203 (size=12301) 2024-11-15T11:52:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671587010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,016 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in a3efb8ce7147f2125466d71154ac42aa of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671587031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671587031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671587040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T11:52:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:07,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T11:52:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671587250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671587250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671587250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T11:52:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:07,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:07,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/e77a6fab8840486ca5f31ea7e5fd49b9 2024-11-15T11:52:07,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a0df92ddded840849c212ca8aa17fb9c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c 2024-11-15T11:52:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:07,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c, entries=250, sequenceid=519, filesize=47.5 K 2024-11-15T11:52:07,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/a5b35d7fb76649f186fab620c8626689 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689 2024-11-15T11:52:07,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689, entries=150, sequenceid=519, filesize=12.0 K 2024-11-15T11:52:07,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/e77a6fab8840486ca5f31ea7e5fd49b9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9 2024-11-15T11:52:07,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9, entries=150, sequenceid=519, filesize=12.0 K 2024-11-15T11:52:07,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~139.50 KB/142848, currentSize=134.18 KB/137400 for a3efb8ce7147f2125466d71154ac42aa in 640ms, sequenceid=519, compaction requested=true 2024-11-15T11:52:07,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:07,419 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:07,419 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:07,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:07,420 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120530 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:07,420 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:07,420 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:07,420 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:07,420 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,420 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,420 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/0c830f61d8b14b31bfeeb659d9f7fbab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=36.6 K 2024-11-15T11:52:07,420 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=117.7 K 2024-11-15T11:52:07,420 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,421 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c] 2024-11-15T11:52:07,421 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 902bdca6b18240c0b34d776bb627df7c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671523331 2024-11-15T11:52:07,421 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c830f61d8b14b31bfeeb659d9f7fbab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671523331 2024-11-15T11:52:07,422 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 653420dc2c544e6e959a3642b988815d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731671525309 2024-11-15T11:52:07,422 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 814dceced81445bfb0b455ab36e1fd65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731671525309 2024-11-15T11:52:07,422 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b35d7fb76649f186fab620c8626689, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:07,422 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0df92ddded840849c212ca8aa17fb9c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:07,429 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:07,433 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:07,433 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/f86e664fdfea400db82ac19a3d34a910 is 50, key is test_row_0/B:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:07,435 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111561e14df0ae3f4f1ea213ffb7808a20f6_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:07,438 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111561e14df0ae3f4f1ea213ffb7808a20f6_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:07,438 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111561e14df0ae3f4f1ea213ffb7808a20f6_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:07,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742029_1205 (size=4469) 2024-11-15T11:52:07,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742028_1204 (size=12983) 2024-11-15T11:52:07,448 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#168 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:07,448 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0310eb87c6ee4dd0bfc794a02bf84458 is 175, key is test_row_0/A:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:07,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742030_1206 (size=31937) 2024-11-15T11:52:07,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T11:52:07,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:07,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:07,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115332f12795785403bb5427eb3d26398f9_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671526913/Put/seqid=0 2024-11-15T11:52:07,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742031_1207 (size=12454) 2024-11-15T11:52:07,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671587563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671587566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671587566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671587567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671587671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671587680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671587680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671587680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,850 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/f86e664fdfea400db82ac19a3d34a910 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/f86e664fdfea400db82ac19a3d34a910 2024-11-15T11:52:07,856 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into f86e664fdfea400db82ac19a3d34a910(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:07,856 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:07,856 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=13, startTime=1731671527419; duration=0sec 2024-11-15T11:52:07,856 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:07,856 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:07,856 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:07,858 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:07,858 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:07,858 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:07,858 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/083efb1262af451781f2482daf5ad7e2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=36.6 K 2024-11-15T11:52:07,859 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 083efb1262af451781f2482daf5ad7e2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671523331 2024-11-15T11:52:07,859 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8afe4e49025143179b1987dadeaef39c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731671525309 2024-11-15T11:52:07,860 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e77a6fab8840486ca5f31ea7e5fd49b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:07,866 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/0310eb87c6ee4dd0bfc794a02bf84458 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458 2024-11-15T11:52:07,869 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:07,870 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/51f21e9db5d94984a1d267fdd6045857 is 50, key is test_row_0/C:col10/1731671526685/Put/seqid=0 2024-11-15T11:52:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671587881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,882 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 0310eb87c6ee4dd0bfc794a02bf84458(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:07,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:07,883 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=13, startTime=1731671527418; duration=0sec 2024-11-15T11:52:07,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:07,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742032_1208 (size=12983) 2024-11-15T11:52:07,890 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/51f21e9db5d94984a1d267fdd6045857 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/51f21e9db5d94984a1d267fdd6045857 2024-11-15T11:52:07,895 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into 51f21e9db5d94984a1d267fdd6045857(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:07,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:07,895 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=13, startTime=1731671527419; duration=0sec 2024-11-15T11:52:07,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:07,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:07,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671587890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671587890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671587890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:07,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:07,960 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115332f12795785403bb5427eb3d26398f9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115332f12795785403bb5427eb3d26398f9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a6174d3346a94c7582e3f089a8538721, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a6174d3346a94c7582e3f089a8538721 is 175, key is test_row_0/A:col10/1731671526913/Put/seqid=0 2024-11-15T11:52:07,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742033_1209 (size=31255) 2024-11-15T11:52:08,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671588190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671588201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671588210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671588210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,377 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=556, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a6174d3346a94c7582e3f089a8538721 2024-11-15T11:52:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/747d9a03da0f43369dc654e7d9b3ac36 is 50, key is test_row_0/B:col10/1731671526913/Put/seqid=0 2024-11-15T11:52:08,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742034_1210 (size=12301) 2024-11-15T11:52:08,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/747d9a03da0f43369dc654e7d9b3ac36 2024-11-15T11:52:08,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/3f93a1f24cb04a7c9b754da630791887 is 50, key is test_row_0/C:col10/1731671526913/Put/seqid=0 2024-11-15T11:52:08,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742035_1211 (size=12301) 2024-11-15T11:52:08,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671588700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671588720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671588720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671588720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:08,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:08,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=556 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/3f93a1f24cb04a7c9b754da630791887 2024-11-15T11:52:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/a6174d3346a94c7582e3f089a8538721 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721 2024-11-15T11:52:08,846 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721, entries=150, sequenceid=556, filesize=30.5 K 2024-11-15T11:52:08,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/747d9a03da0f43369dc654e7d9b3ac36 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36 2024-11-15T11:52:08,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36, entries=150, sequenceid=556, filesize=12.0 K 2024-11-15T11:52:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/3f93a1f24cb04a7c9b754da630791887 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887 2024-11-15T11:52:08,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887, entries=150, sequenceid=556, filesize=12.0 K 2024-11-15T11:52:08,864 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for a3efb8ce7147f2125466d71154ac42aa in 1323ms, sequenceid=556, compaction requested=false 2024-11-15T11:52:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:08,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T11:52:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T11:52:08,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T11:52:08,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0920 sec 2024-11-15T11:52:08,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0970 sec 2024-11-15T11:52:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T11:52:08,920 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:08,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T11:52:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T11:52:08,923 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:08,924 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:08,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T11:52:09,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T11:52:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:09,078 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:09,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:09,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a707484e062d43cda01e1acc2ffdf1ef_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671527564/Put/seqid=0 2024-11-15T11:52:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742036_1212 (size=12454) 2024-11-15T11:52:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:09,096 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a707484e062d43cda01e1acc2ffdf1ef_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a707484e062d43cda01e1acc2ffdf1ef_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4e43cb514b4f4f0ea8d3b477032b2423, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:09,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4e43cb514b4f4f0ea8d3b477032b2423 is 175, key is test_row_0/A:col10/1731671527564/Put/seqid=0 2024-11-15T11:52:09,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742037_1213 (size=31255) 2024-11-15T11:52:09,101 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=569, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4e43cb514b4f4f0ea8d3b477032b2423 2024-11-15T11:52:09,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c3e66c36edf14253bb8efc6e7eb06c60 is 50, key is test_row_0/B:col10/1731671527564/Put/seqid=0 2024-11-15T11:52:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742038_1214 (size=12301) 2024-11-15T11:52:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T11:52:09,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c3e66c36edf14253bb8efc6e7eb06c60 2024-11-15T11:52:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/2154f3e9ff4d4c2fa86afe097dde04d0 is 50, key is test_row_0/C:col10/1731671527564/Put/seqid=0 2024-11-15T11:52:09,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742039_1215 (size=12301) 2024-11-15T11:52:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T11:52:09,572 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:52:09,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731671529741, max timestamp=1731671529755Num uniques 859; 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731671529741, max timestamp=1731671529755Num uniques 859; 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731671529741, max timestamp=1731671529755Num uniques 859; 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:52:09,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:52:09,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671589785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671589787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671589794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671589795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671589901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671589910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:09,949 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/2154f3e9ff4d4c2fa86afe097dde04d0 2024-11-15T11:52:09,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4e43cb514b4f4f0ea8d3b477032b2423 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423 2024-11-15T11:52:09,965 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423, entries=150, sequenceid=569, filesize=30.5 K 2024-11-15T11:52:09,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c3e66c36edf14253bb8efc6e7eb06c60 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60 2024-11-15T11:52:09,980 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60, entries=150, sequenceid=569, filesize=12.0 K 2024-11-15T11:52:09,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/2154f3e9ff4d4c2fa86afe097dde04d0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0 2024-11-15T11:52:09,990 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0, entries=150, sequenceid=569, filesize=12.0 K 2024-11-15T11:52:09,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a3efb8ce7147f2125466d71154ac42aa in 915ms, sequenceid=569, compaction requested=true 2024-11-15T11:52:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:09,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T11:52:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T11:52:09,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T11:52:09,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-15T11:52:10,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.0770 sec 2024-11-15T11:52:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T11:52:10,061 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:10,061 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T11:52:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:10,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:10,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:10,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:10,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:10,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:10,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115709c00b61dd04ab4a5a792eb3a1fdd6e_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671590137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671590137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671590138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671590140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742040_1216 (size=14994) 2024-11-15T11:52:10,152 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:10,158 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115709c00b61dd04ab4a5a792eb3a1fdd6e_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115709c00b61dd04ab4a5a792eb3a1fdd6e_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:10,159 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4c54f8993e5d4041ab68e27029612d93, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:10,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4c54f8993e5d4041ab68e27029612d93 is 175, key is test_row_0/A:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:10,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742041_1217 (size=39949) 2024-11-15T11:52:10,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=605, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4c54f8993e5d4041ab68e27029612d93 2024-11-15T11:52:10,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e9fe0d2c93514c6bbc4e0815f3a54556 is 50, key is test_row_0/B:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:10,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742042_1218 (size=12301) 2024-11-15T11:52:10,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671590250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671590250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671590251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671590251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671590460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671590461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671590461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671590461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e9fe0d2c93514c6bbc4e0815f3a54556 2024-11-15T11:52:10,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f80947da26a346708c6cb961e353e220 is 50, key is test_row_0/C:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:10,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742043_1219 (size=12301) 2024-11-15T11:52:10,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671590770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671590770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671590771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671590771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:10,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:10,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:10,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:10,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:11,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=605 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f80947da26a346708c6cb961e353e220 2024-11-15T11:52:11,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/4c54f8993e5d4041ab68e27029612d93 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93 2024-11-15T11:52:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43766 deadline: 1731671591040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,045 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in a3efb8ce7147f2125466d71154ac42aa of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93, entries=200, sequenceid=605, filesize=39.0 K 2024-11-15T11:52:11,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/e9fe0d2c93514c6bbc4e0815f3a54556 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556 2024-11-15T11:52:11,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556, entries=150, sequenceid=605, filesize=12.0 K 2024-11-15T11:52:11,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f80947da26a346708c6cb961e353e220 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220 2024-11-15T11:52:11,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220, entries=150, sequenceid=605, filesize=12.0 K 2024-11-15T11:52:11,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for a3efb8ce7147f2125466d71154ac42aa in 954ms, sequenceid=605, compaction requested=true 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:11,076 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:11,076 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:11,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:11,077 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:11,077 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:11,077 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:11,078 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/f86e664fdfea400db82ac19a3d34a910, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=48.7 K 2024-11-15T11:52:11,078 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:11,078 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:11,078 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:11,078 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=131.2 K 2024-11-15T11:52:11,078 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:11,078 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93] 2024-11-15T11:52:11,079 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f86e664fdfea400db82ac19a3d34a910, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:11,079 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0310eb87c6ee4dd0bfc794a02bf84458, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:11,079 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6174d3346a94c7582e3f089a8538721, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731671526892 2024-11-15T11:52:11,080 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 747d9a03da0f43369dc654e7d9b3ac36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731671526892 2024-11-15T11:52:11,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e43cb514b4f4f0ea8d3b477032b2423, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731671527564 2024-11-15T11:52:11,080 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e66c36edf14253bb8efc6e7eb06c60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731671527564 2024-11-15T11:52:11,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c54f8993e5d4041ab68e27029612d93, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529785 2024-11-15T11:52:11,080 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fe0d2c93514c6bbc4e0815f3a54556, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529787 2024-11-15T11:52:11,093 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:11,103 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:11,104 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c7349c0e5ace401590c30bf7ac42b8c9 is 50, key is test_row_0/B:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:11,106 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157f441980b612466ab8ef2cfb0af7c10d_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:11,109 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157f441980b612466ab8ef2cfb0af7c10d_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:11,109 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157f441980b612466ab8ef2cfb0af7c10d_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:11,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742044_1220 (size=13119) 2024-11-15T11:52:11,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742045_1221 (size=4469) 2024-11-15T11:52:11,122 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#180 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:11,123 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/791a9a459bf9496eae810e4267dc1815 is 175, key is test_row_0/A:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:11,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742046_1222 (size=32073) 2024-11-15T11:52:11,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:11,138 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:11,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:11,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfdd48bb91a0416fb085627946e8e1c9_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671530136/Put/seqid=0 2024-11-15T11:52:11,149 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/791a9a459bf9496eae810e4267dc1815 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815 2024-11-15T11:52:11,158 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 791a9a459bf9496eae810e4267dc1815(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:11,158 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:11,158 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=12, startTime=1731671531076; duration=0sec 2024-11-15T11:52:11,158 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:11,158 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:11,158 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:11,161 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:11,161 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:11,161 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:11,161 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/51f21e9db5d94984a1d267fdd6045857, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=48.7 K 2024-11-15T11:52:11,162 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f21e9db5d94984a1d267fdd6045857, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731671526685 2024-11-15T11:52:11,162 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f93a1f24cb04a7c9b754da630791887, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=556, earliestPutTs=1731671526892 2024-11-15T11:52:11,162 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2154f3e9ff4d4c2fa86afe097dde04d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731671527564 2024-11-15T11:52:11,163 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80947da26a346708c6cb961e353e220, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529787 2024-11-15T11:52:11,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742047_1223 (size=12454) 2024-11-15T11:52:11,170 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:11,170 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/852955baba344e69bd6c375f7b5382bb is 50, key is test_row_0/C:col10/1731671530116/Put/seqid=0 2024-11-15T11:52:11,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742048_1224 (size=13119) 2024-11-15T11:52:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:11,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:11,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671531290, max timestamp=1731671531305Num uniques 859; 2024-11-15T11:52:11,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:11,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671531290, max timestamp=1731671531305Num uniques 859; 2024-11-15T11:52:11,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:11,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671531290, max timestamp=1731671531305Num uniques 859; 2024-11-15T11:52:11,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:11,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671591331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671591379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671591379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671591379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,394 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671591490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671591500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671591500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671591500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,516 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/c7349c0e5ace401590c30bf7ac42b8c9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c7349c0e5ace401590c30bf7ac42b8c9 2024-11-15T11:52:11,521 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into c7349c0e5ace401590c30bf7ac42b8c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:11,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:11,521 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=12, startTime=1731671531076; duration=0sec 2024-11-15T11:52:11,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:11,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:11,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:11,568 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bfdd48bb91a0416fb085627946e8e1c9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfdd48bb91a0416fb085627946e8e1c9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b72dd327b4b64ddca3d1edd3931f69fa, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:11,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b72dd327b4b64ddca3d1edd3931f69fa is 175, key is test_row_0/A:col10/1731671530136/Put/seqid=0 2024-11-15T11:52:11,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742049_1225 (size=31255) 2024-11-15T11:52:11,579 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/852955baba344e69bd6c375f7b5382bb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/852955baba344e69bd6c375f7b5382bb 2024-11-15T11:52:11,588 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into 852955baba344e69bd6c375f7b5382bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:11,588 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:11,588 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=12, startTime=1731671531076; duration=0sec 2024-11-15T11:52:11,588 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:11,588 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:11,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671591700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671591710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671591711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671591711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:11,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=615, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b72dd327b4b64ddca3d1edd3931f69fa 2024-11-15T11:52:11,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2a3d247a4b0542a2b5295feab63204cf is 50, key is test_row_0/B:col10/1731671530136/Put/seqid=0 2024-11-15T11:52:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742050_1226 (size=12301) 2024-11-15T11:52:12,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2a3d247a4b0542a2b5295feab63204cf 2024-11-15T11:52:12,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671592010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/d2975fce0ee941b38d97d204bebbcc14 is 50, key is test_row_0/C:col10/1731671530136/Put/seqid=0 2024-11-15T11:52:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671592021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671592021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742051_1227 (size=12301) 2024-11-15T11:52:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/d2975fce0ee941b38d97d204bebbcc14 2024-11-15T11:52:12,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/b72dd327b4b64ddca3d1edd3931f69fa as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa 2024-11-15T11:52:12,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa, entries=150, sequenceid=615, filesize=30.5 K 2024-11-15T11:52:12,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/2a3d247a4b0542a2b5295feab63204cf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf 2024-11-15T11:52:12,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf, entries=150, sequenceid=615, filesize=12.0 K 2024-11-15T11:52:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/d2975fce0ee941b38d97d204bebbcc14 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14 2024-11-15T11:52:12,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14, entries=150, sequenceid=615, filesize=12.0 K 2024-11-15T11:52:12,074 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a3efb8ce7147f2125466d71154ac42aa in 936ms, sequenceid=615, compaction requested=false 2024-11-15T11:52:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T11:52:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T11:52:12,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T11:52:12,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-11-15T11:52:12,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0160 sec 2024-11-15T11:52:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T11:52:12,210 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:12,211 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-15T11:52:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T11:52:12,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:12,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:12,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T11:52:12,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,365 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a408af4c74974f5caac5f16cf44889c0_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671531333/Put/seqid=0 2024-11-15T11:52:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742052_1228 (size=12454) 2024-11-15T11:52:12,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,393 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a408af4c74974f5caac5f16cf44889c0_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a408af4c74974f5caac5f16cf44889c0_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/021c66c730c44ae2aabd326f241f1128, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:12,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/021c66c730c44ae2aabd326f241f1128 is 175, key is test_row_0/A:col10/1731671531333/Put/seqid=0 2024-11-15T11:52:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742053_1229 (size=31255) 2024-11-15T11:52:12,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=653, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/021c66c730c44ae2aabd326f241f1128 2024-11-15T11:52:12,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/94966db01d0c44c49e6a0ad191bed32a is 50, key is test_row_0/B:col10/1731671531333/Put/seqid=0 2024-11-15T11:52:12,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742054_1230 (size=12301) 2024-11-15T11:52:12,414 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/94966db01d0c44c49e6a0ad191bed32a 2024-11-15T11:52:12,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f1f6680c22bc43258a78a3771248fd75 is 50, key is test_row_0/C:col10/1731671531333/Put/seqid=0 2024-11-15T11:52:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742055_1231 (size=12301) 2024-11-15T11:52:12,428 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=653 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f1f6680c22bc43258a78a3771248fd75 2024-11-15T11:52:12,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/021c66c730c44ae2aabd326f241f1128 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128 2024-11-15T11:52:12,436 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128, entries=150, sequenceid=653, filesize=30.5 K 2024-11-15T11:52:12,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/94966db01d0c44c49e6a0ad191bed32a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a 2024-11-15T11:52:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,441 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a, entries=150, sequenceid=653, filesize=12.0 K 2024-11-15T11:52:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/f1f6680c22bc43258a78a3771248fd75 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75 2024-11-15T11:52:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,449 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75, entries=150, sequenceid=653, filesize=12.0 K 2024-11-15T11:52:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=0 B/0 for a3efb8ce7147f2125466d71154ac42aa in 85ms, sequenceid=653, compaction requested=true 2024-11-15T11:52:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T11:52:12,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 238 msec 2024-11-15T11:52:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 243 msec 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T11:52:12,531 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-15T11:52:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,534 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,535 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:12,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6df7bf833d844209749cf7f8a6fc0b7_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742056_1232 (size=22618) 2024-11-15T11:52:12,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:12,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6df7bf833d844209749cf7f8a6fc0b7_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6df7bf833d844209749cf7f8a6fc0b7_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:12,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/bc853619afcf48fbb7b29766145d3cde, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:12,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/bc853619afcf48fbb7b29766145d3cde is 175, key is test_row_0/A:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:12,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671532655, max timestamp=1731671532673Num uniques 859; 2024-11-15T11:52:12,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:12,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671532655, max timestamp=1731671532673Num uniques 859; 2024-11-15T11:52:12,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:12,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671532655, max timestamp=1731671532673Num uniques 859; 2024-11-15T11:52:12,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:12,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:12,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:12,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742057_1233 (size=66023) 2024-11-15T11:52:12,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671592711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671592712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671592721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671592723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671592831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:12,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:12,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:12,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:12,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671593041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671593042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671593042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671593044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=663, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/bc853619afcf48fbb7b29766145d3cde 2024-11-15T11:52:13,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/53c8376ed445478d9cc2b170fd520804 is 50, key is test_row_0/B:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:13,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742058_1234 (size=12301) 2024-11-15T11:52:13,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:13,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671593360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671593360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671593361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671593361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/53c8376ed445478d9cc2b170fd520804 2024-11-15T11:52:13,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/b4489e80432e4c0b92db8ce21893723b is 50, key is test_row_0/C:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:13,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742059_1235 (size=12301) 2024-11-15T11:52:13,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:13,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43744 deadline: 1731671593870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43750 deadline: 1731671593871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43704 deadline: 1731671593871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43710 deadline: 1731671593880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:13,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:13,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:13,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:13,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:13,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=663 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/b4489e80432e4c0b92db8ce21893723b 2024-11-15T11:52:13,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/bc853619afcf48fbb7b29766145d3cde as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde 2024-11-15T11:52:13,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde, entries=350, sequenceid=663, filesize=64.5 K 2024-11-15T11:52:13,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/53c8376ed445478d9cc2b170fd520804 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804 2024-11-15T11:52:13,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804, entries=150, sequenceid=663, filesize=12.0 K 2024-11-15T11:52:13,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/b4489e80432e4c0b92db8ce21893723b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b 2024-11-15T11:52:13,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b, entries=150, sequenceid=663, filesize=12.0 K 2024-11-15T11:52:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a3efb8ce7147f2125466d71154ac42aa in 1364ms, sequenceid=663, compaction requested=true 2024-11-15T11:52:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:13,983 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:13,983 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a3efb8ce7147f2125466d71154ac42aa:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:13,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:13,986 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:13,987 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/B is initiating minor compaction (all files) 2024-11-15T11:52:13,987 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/B in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,987 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c7349c0e5ace401590c30bf7ac42b8c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=48.8 K 2024-11-15T11:52:13,987 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160606 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:13,987 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/A is initiating minor compaction (all files) 2024-11-15T11:52:13,987 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/A in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,987 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=156.8 K 2024-11-15T11:52:13,987 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:13,987 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde] 2024-11-15T11:52:13,988 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c7349c0e5ace401590c30bf7ac42b8c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529787 2024-11-15T11:52:13,988 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 791a9a459bf9496eae810e4267dc1815, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529787 2024-11-15T11:52:13,988 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3d247a4b0542a2b5295feab63204cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1731671530135 2024-11-15T11:52:13,988 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72dd327b4b64ddca3d1edd3931f69fa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1731671530135 2024-11-15T11:52:13,988 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 94966db01d0c44c49e6a0ad191bed32a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731671531327 2024-11-15T11:52:13,989 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 021c66c730c44ae2aabd326f241f1128, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731671531327 2024-11-15T11:52:13,989 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 53c8376ed445478d9cc2b170fd520804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731671532580 2024-11-15T11:52:13,989 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc853619afcf48fbb7b29766145d3cde, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731671532578 2024-11-15T11:52:14,001 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:52:14,002 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/758fb3c0736942ca9247516222e42d00 is 50, key is test_row_0/B:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:14,003 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:14,010 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115808826a916a54ad39dc5eef6463380e6_a3efb8ce7147f2125466d71154ac42aa store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:14,012 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115808826a916a54ad39dc5eef6463380e6_a3efb8ce7147f2125466d71154ac42aa, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:14,013 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115808826a916a54ad39dc5eef6463380e6_a3efb8ce7147f2125466d71154ac42aa because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:14,014 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-15T11:52:14,014 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,014 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,014 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,015 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-15T11:52:14,015 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,015 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,015 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,015 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,016 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-15T11:52:14,016 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,016 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,016 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742060_1236 (size=13255) 2024-11-15T11:52:14,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742061_1237 (size=4469) 2024-11-15T11:52:14,031 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-15T11:52:14,031 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,031 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,031 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-15T11:52:14,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:14,069 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:14,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:14,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115614f928a156344c5a36403ce874988a5_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671532699/Put/seqid=0 2024-11-15T11:52:14,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742062_1238 (size=12454) 2024-11-15T11:52:14,213 DEBUG [regionserver/7adf9b3d9d04:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-15T11:52:14,425 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:14,426 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/059edbd2641d46f8be1caf3031b4489f is 175, key is test_row_0/A:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:14,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742063_1239 (size=32209) 2024-11-15T11:52:14,435 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/758fb3c0736942ca9247516222e42d00 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/758fb3c0736942ca9247516222e42d00 2024-11-15T11:52:14,440 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/059edbd2641d46f8be1caf3031b4489f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/059edbd2641d46f8be1caf3031b4489f 2024-11-15T11:52:14,441 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/B of a3efb8ce7147f2125466d71154ac42aa into 758fb3c0736942ca9247516222e42d00(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:14,441 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:14,441 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/B, priority=12, startTime=1731671533983; duration=0sec 2024-11-15T11:52:14,441 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:14,441 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:B 2024-11-15T11:52:14,441 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:14,443 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:14,443 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): a3efb8ce7147f2125466d71154ac42aa/C is initiating minor compaction (all files) 2024-11-15T11:52:14,443 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a3efb8ce7147f2125466d71154ac42aa/C in TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:14,443 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/852955baba344e69bd6c375f7b5382bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp, totalSize=48.8 K 2024-11-15T11:52:14,444 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 852955baba344e69bd6c375f7b5382bb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=605, earliestPutTs=1731671529787 2024-11-15T11:52:14,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d2975fce0ee941b38d97d204bebbcc14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1731671530135 2024-11-15T11:52:14,446 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f6680c22bc43258a78a3771248fd75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=653, earliestPutTs=1731671531327 2024-11-15T11:52:14,446 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/A of a3efb8ce7147f2125466d71154ac42aa into 059edbd2641d46f8be1caf3031b4489f(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:14,446 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:14,446 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/A, priority=12, startTime=1731671533983; duration=0sec 2024-11-15T11:52:14,446 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:14,446 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:A 2024-11-15T11:52:14,446 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b4489e80432e4c0b92db8ce21893723b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=663, earliestPutTs=1731671532580 2024-11-15T11:52:14,457 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a3efb8ce7147f2125466d71154ac42aa#C#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:14,458 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/2bf5a89039584533b7fd97c3a73cc684 is 50, key is test_row_0/C:col10/1731671532617/Put/seqid=0 2024-11-15T11:52:14,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742064_1240 (size=13255) 2024-11-15T11:52:14,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:14,503 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115614f928a156344c5a36403ce874988a5_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115614f928a156344c5a36403ce874988a5_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/508bd51eb5b743749412b6c425677b67, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/508bd51eb5b743749412b6c425677b67 is 175, key is test_row_0/A:col10/1731671532699/Put/seqid=0 2024-11-15T11:52:14,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742065_1241 (size=31255) 2024-11-15T11:52:14,517 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=699, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/508bd51eb5b743749412b6c425677b67 2024-11-15T11:52:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/fe56ee30d8f24e7ab26067685b2393af is 50, key is test_row_0/B:col10/1731671532699/Put/seqid=0 2024-11-15T11:52:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742066_1242 (size=12301) 2024-11-15T11:52:14,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/fe56ee30d8f24e7ab26067685b2393af 2024-11-15T11:52:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/ef09ce7b2f9944079c6874262fe1f61f is 50, key is test_row_0/C:col10/1731671532699/Put/seqid=0 2024-11-15T11:52:14,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742067_1243 (size=12301) 2024-11-15T11:52:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:14,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:14,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. as already flushing 2024-11-15T11:52:14,882 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-15T11:52:14,882 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,882 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,882 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,893 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-15T11:52:14,893 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,893 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,893 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,894 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-15T11:52:14,894 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,894 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,895 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,897 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/2bf5a89039584533b7fd97c3a73cc684 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2bf5a89039584533b7fd97c3a73cc684 2024-11-15T11:52:14,897 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-15T11:52:14,897 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:14,897 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,898 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:14,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:14,904 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a3efb8ce7147f2125466d71154ac42aa/C of a3efb8ce7147f2125466d71154ac42aa into 2bf5a89039584533b7fd97c3a73cc684(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:14,904 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:14,904 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., storeName=a3efb8ce7147f2125466d71154ac42aa/C, priority=12, startTime=1731671533983; duration=0sec 2024-11-15T11:52:14,904 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:14,904 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a3efb8ce7147f2125466d71154ac42aa:C 2024-11-15T11:52:14,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=699 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/ef09ce7b2f9944079c6874262fe1f61f 2024-11-15T11:52:14,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/508bd51eb5b743749412b6c425677b67 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/508bd51eb5b743749412b6c425677b67 2024-11-15T11:52:14,967 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/508bd51eb5b743749412b6c425677b67, entries=150, sequenceid=699, filesize=30.5 K 2024-11-15T11:52:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/fe56ee30d8f24e7ab26067685b2393af as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe56ee30d8f24e7ab26067685b2393af 2024-11-15T11:52:14,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe56ee30d8f24e7ab26067685b2393af, entries=150, sequenceid=699, filesize=12.0 K 2024-11-15T11:52:14,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/ef09ce7b2f9944079c6874262fe1f61f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/ef09ce7b2f9944079c6874262fe1f61f 2024-11-15T11:52:14,983 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/ef09ce7b2f9944079c6874262fe1f61f, entries=150, sequenceid=699, filesize=12.0 K 2024-11-15T11:52:14,984 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=26.84 KB/27480 for a3efb8ce7147f2125466d71154ac42aa in 915ms, sequenceid=699, compaction requested=false 2024-11-15T11:52:14,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for a3efb8ce7147f2125466d71154ac42aa: 2024-11-15T11:52:14,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:14,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-15T11:52:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-15T11:52:14,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T11:52:14,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4510 sec 2024-11-15T11:52:14,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.4580 sec 2024-11-15T11:52:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T11:52:16,690 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:21,112 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-15T11:52:21,113 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:21,113 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:21,113 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:21,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2440 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2606 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1119 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3355 rows 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1122 2024-11-15T11:52:21,113 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3359 rows 2024-11-15T11:52:21,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:52:21,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:52:21,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:21,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:21,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:21,114 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:52:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:21,119 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671541118"}]},"ts":"1731671541118"} 2024-11-15T11:52:21,120 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:52:21,120 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:52:21,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:52:21,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, UNASSIGN}] 2024-11-15T11:52:21,125 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, UNASSIGN 2024-11-15T11:52:21,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:21,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, UNASSIGN because future has completed 2024-11-15T11:52:21,130 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:52:21,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:21,284 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:21,284 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:52:21,284 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing a3efb8ce7147f2125466d71154ac42aa, disabling compactions & flushes 2024-11-15T11:52:21,285 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. after waiting 0 ms 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:21,285 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing a3efb8ce7147f2125466d71154ac42aa 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=A 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=B 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a3efb8ce7147f2125466d71154ac42aa, store=C 2024-11-15T11:52:21,285 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:21,293 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150ece1b6c58044d939694481ae5c94af0_a3efb8ce7147f2125466d71154ac42aa is 50, key is test_row_0/A:col10/1731671534891/Put/seqid=0 2024-11-15T11:52:21,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742068_1244 (size=12454) 2024-11-15T11:52:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:21,698 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:21,704 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150ece1b6c58044d939694481ae5c94af0_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150ece1b6c58044d939694481ae5c94af0_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:21,705 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/2a2adbbd8b7448a7b972874aa6969eb7, store: [table=TestAcidGuarantees family=A region=a3efb8ce7147f2125466d71154ac42aa] 2024-11-15T11:52:21,706 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/2a2adbbd8b7448a7b972874aa6969eb7 is 175, key is test_row_0/A:col10/1731671534891/Put/seqid=0 2024-11-15T11:52:21,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742069_1245 (size=31255) 2024-11-15T11:52:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:22,111 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=710, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/2a2adbbd8b7448a7b972874aa6969eb7 2024-11-15T11:52:22,153 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/5ae148f1cc164abebc3c8552d07e549e is 50, key is test_row_0/B:col10/1731671534891/Put/seqid=0 2024-11-15T11:52:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742070_1246 (size=12301) 2024-11-15T11:52:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:22,560 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/5ae148f1cc164abebc3c8552d07e549e 2024-11-15T11:52:22,569 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8a589d6141c24e35a63c55c62d944a03 is 50, key is test_row_0/C:col10/1731671534891/Put/seqid=0 2024-11-15T11:52:22,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742071_1247 (size=12301) 2024-11-15T11:52:22,975 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=710 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8a589d6141c24e35a63c55c62d944a03 2024-11-15T11:52:22,979 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/A/2a2adbbd8b7448a7b972874aa6969eb7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/2a2adbbd8b7448a7b972874aa6969eb7 2024-11-15T11:52:22,983 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/2a2adbbd8b7448a7b972874aa6969eb7, entries=150, sequenceid=710, filesize=30.5 K 2024-11-15T11:52:22,983 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/B/5ae148f1cc164abebc3c8552d07e549e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/5ae148f1cc164abebc3c8552d07e549e 2024-11-15T11:52:22,987 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/5ae148f1cc164abebc3c8552d07e549e, entries=150, sequenceid=710, filesize=12.0 K 2024-11-15T11:52:22,987 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/.tmp/C/8a589d6141c24e35a63c55c62d944a03 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8a589d6141c24e35a63c55c62d944a03 2024-11-15T11:52:22,990 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8a589d6141c24e35a63c55c62d944a03, entries=150, sequenceid=710, filesize=12.0 K 2024-11-15T11:52:22,991 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a3efb8ce7147f2125466d71154ac42aa in 1706ms, sequenceid=710, compaction requested=true 2024-11-15T11:52:22,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde] to archive 2024-11-15T11:52:22,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:22,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/ff1c8b69988d44eab1ecdb1f32c426fe 2024-11-15T11:52:22,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/baf6f198029b4c3497510c9b644e2483 2024-11-15T11:52:22,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a9904996d4b8460ab34d1cba92473910 2024-11-15T11:52:23,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/c597ec5a4f004eecb4c723bbc65d54e3 2024-11-15T11:52:23,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/61ffb62805f748a09835017133bd42dc 2024-11-15T11:52:23,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0262ce919a3042708b69b4faa74b97ea 2024-11-15T11:52:23,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/3f63fca16ca94d6a9a43572fff883f91 2024-11-15T11:52:23,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b3a7a46745bb4e5993e6d24cc7cc0278 2024-11-15T11:52:23,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5964db4c151645c48667e0a65e4a1d29 2024-11-15T11:52:23,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/96e0bcae5ef74871bc0c6e7e58a0360a 2024-11-15T11:52:23,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/5053f7cc99c44fd89f44e60ac68ed2f6 2024-11-15T11:52:23,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/e05bd528e53349f5a876348d3e9664cb 2024-11-15T11:52:23,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/9b03468af6454f5eb54c254e62e839e2 2024-11-15T11:52:23,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/21c0ff5054144b168feaea7c20a51bbd 2024-11-15T11:52:23,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4af60d9caf80481f86d54b843ed30e07 2024-11-15T11:52:23,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/1c24851f52154572a88d96ba72da3ea0 2024-11-15T11:52:23,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/902bdca6b18240c0b34d776bb627df7c 2024-11-15T11:52:23,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/787409e4cee449d2822950ba09c3610d 2024-11-15T11:52:23,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/653420dc2c544e6e959a3642b988815d 2024-11-15T11:52:23,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a0df92ddded840849c212ca8aa17fb9c 2024-11-15T11:52:23,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/0310eb87c6ee4dd0bfc794a02bf84458 2024-11-15T11:52:23,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/a6174d3346a94c7582e3f089a8538721 2024-11-15T11:52:23,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4e43cb514b4f4f0ea8d3b477032b2423 2024-11-15T11:52:23,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/4c54f8993e5d4041ab68e27029612d93 2024-11-15T11:52:23,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/791a9a459bf9496eae810e4267dc1815 2024-11-15T11:52:23,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/b72dd327b4b64ddca3d1edd3931f69fa 2024-11-15T11:52:23,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/021c66c730c44ae2aabd326f241f1128 2024-11-15T11:52:23,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/bc853619afcf48fbb7b29766145d3cde 2024-11-15T11:52:23,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe260f5e6c1a4a6385810e0c3e9695c1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c09c3a0ae3ef4890a0dabe4c33cc249e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/62cff8bb64c0485f99058dab712fb187, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/243ac8f3447849b182ad44b1f829685f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/0c830f61d8b14b31bfeeb659d9f7fbab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/f86e664fdfea400db82ac19a3d34a910, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c7349c0e5ace401590c30bf7ac42b8c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804] to archive 2024-11-15T11:52:23,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:23,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/dd3fa20da34243dbbc76f0e03866a350 2024-11-15T11:52:23,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6241e76ff27c426d8f6859a53aebe373 2024-11-15T11:52:23,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/ec5b12662a874af48778ca5eb6d07505 2024-11-15T11:52:23,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe260f5e6c1a4a6385810e0c3e9695c1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe260f5e6c1a4a6385810e0c3e9695c1 2024-11-15T11:52:23,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/8262e85aee0340bd856ec375769cfbdc 2024-11-15T11:52:23,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/7e1b84522e564d0097c36ecf942e64f2 2024-11-15T11:52:23,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c09c3a0ae3ef4890a0dabe4c33cc249e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c09c3a0ae3ef4890a0dabe4c33cc249e 2024-11-15T11:52:23,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6da1b98d2ca3446b8ca58a5b3fea4a14 2024-11-15T11:52:23,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2d264f0fd5294849a2d998b32c94ce35 2024-11-15T11:52:23,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/62cff8bb64c0485f99058dab712fb187 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/62cff8bb64c0485f99058dab712fb187 2024-11-15T11:52:23,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/86d8f7db34814cbbb22ef71d5fb68bd6 2024-11-15T11:52:23,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/9b56b3501dec4e5c9765362599fa38c4 2024-11-15T11:52:23,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/243ac8f3447849b182ad44b1f829685f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/243ac8f3447849b182ad44b1f829685f 2024-11-15T11:52:23,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a699ba72897c41b9afa8f58fc0a8fd62 2024-11-15T11:52:23,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/6aaf74efd89a423590db8b2f3cc2bee9 2024-11-15T11:52:23,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/be3c5bd65e404ed69f7a1dd71e622679 2024-11-15T11:52:23,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/0c830f61d8b14b31bfeeb659d9f7fbab to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/0c830f61d8b14b31bfeeb659d9f7fbab 2024-11-15T11:52:23,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e078573f05904968986e55512479ba02 2024-11-15T11:52:23,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/814dceced81445bfb0b455ab36e1fd65 2024-11-15T11:52:23,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/f86e664fdfea400db82ac19a3d34a910 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/f86e664fdfea400db82ac19a3d34a910 2024-11-15T11:52:23,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/a5b35d7fb76649f186fab620c8626689 2024-11-15T11:52:23,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/747d9a03da0f43369dc654e7d9b3ac36 2024-11-15T11:52:23,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c3e66c36edf14253bb8efc6e7eb06c60 2024-11-15T11:52:23,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c7349c0e5ace401590c30bf7ac42b8c9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/c7349c0e5ace401590c30bf7ac42b8c9 2024-11-15T11:52:23,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/e9fe0d2c93514c6bbc4e0815f3a54556 2024-11-15T11:52:23,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/2a3d247a4b0542a2b5295feab63204cf 2024-11-15T11:52:23,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/94966db01d0c44c49e6a0ad191bed32a 2024-11-15T11:52:23,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/53c8376ed445478d9cc2b170fd520804 2024-11-15T11:52:23,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/eb939d14f263478fb5c267e28fc616fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/c2626e8622e947c09a2859c25153df94, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3a1bb6cd56054d6496760910378c1e8d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e0f4a922e0364bc2a243e0357799114c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/083efb1262af451781f2482daf5ad7e2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/51f21e9db5d94984a1d267fdd6045857, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/852955baba344e69bd6c375f7b5382bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b] to archive 2024-11-15T11:52:23,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:23,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/4350fb6a5cc042abaa458a7fae5e57c9 2024-11-15T11:52:23,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/731984b1bbf04005b6c428f6af619261 2024-11-15T11:52:23,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/db06450041ba4aa58c4f3a6c1e6b9df6 2024-11-15T11:52:23,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/eb939d14f263478fb5c267e28fc616fe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/eb939d14f263478fb5c267e28fc616fe 2024-11-15T11:52:23,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/421cda639a8049f78434764c7fb55faf 2024-11-15T11:52:23,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/13b2d7cb1ece44c2b06c0860f3e8b0fa 2024-11-15T11:52:23,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/c2626e8622e947c09a2859c25153df94 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/c2626e8622e947c09a2859c25153df94 2024-11-15T11:52:23,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6acca095675e47389832615f98af44e0 2024-11-15T11:52:23,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/99c227e9005b4462890f67ec71ddb233 2024-11-15T11:52:23,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3a1bb6cd56054d6496760910378c1e8d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3a1bb6cd56054d6496760910378c1e8d 2024-11-15T11:52:23,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/1aaade8c834b45828fd7979492372cb9 2024-11-15T11:52:23,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8e027e87d6054fd39a833c34e2e15f78 2024-11-15T11:52:23,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e0f4a922e0364bc2a243e0357799114c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e0f4a922e0364bc2a243e0357799114c 2024-11-15T11:52:23,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/acebee9bf65047eab025e832bb848bfa 2024-11-15T11:52:23,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/bbde7898869d4842a7414dae9073e16e 2024-11-15T11:52:23,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/6e1dd049289c4ddaa7fdf1713aa63ab1 2024-11-15T11:52:23,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/083efb1262af451781f2482daf5ad7e2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/083efb1262af451781f2482daf5ad7e2 2024-11-15T11:52:23,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/32d0288f71cf4be4b0ee62d984736466 2024-11-15T11:52:23,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8afe4e49025143179b1987dadeaef39c 2024-11-15T11:52:23,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/51f21e9db5d94984a1d267fdd6045857 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/51f21e9db5d94984a1d267fdd6045857 2024-11-15T11:52:23,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/e77a6fab8840486ca5f31ea7e5fd49b9 2024-11-15T11:52:23,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/3f93a1f24cb04a7c9b754da630791887 2024-11-15T11:52:23,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2154f3e9ff4d4c2fa86afe097dde04d0 2024-11-15T11:52:23,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/852955baba344e69bd6c375f7b5382bb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/852955baba344e69bd6c375f7b5382bb 2024-11-15T11:52:23,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f80947da26a346708c6cb961e353e220 2024-11-15T11:52:23,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/d2975fce0ee941b38d97d204bebbcc14 2024-11-15T11:52:23,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/f1f6680c22bc43258a78a3771248fd75 2024-11-15T11:52:23,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/b4489e80432e4c0b92db8ce21893723b 2024-11-15T11:52:23,116 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits/713.seqid, newMaxSeqId=713, maxSeqId=4 2024-11-15T11:52:23,117 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. 2024-11-15T11:52:23,117 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for a3efb8ce7147f2125466d71154ac42aa: Waiting for close lock at 1731671541284Running coprocessor pre-close hooks at 1731671541284Disabling compacts and flushes for region at 1731671541284Disabling writes for close at 1731671541285 (+1 ms)Obtaining lock to block concurrent updates at 1731671541285Preparing flush snapshotting stores in a3efb8ce7147f2125466d71154ac42aa at 1731671541285Finished memstore snapshotting TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731671541285Flushing stores of TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa. at 1731671541286 (+1 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/A: creating writer at 1731671541286Flushing mob file a3efb8ce7147f2125466d71154ac42aa/A: appending metadata at 1731671541293 (+7 ms)Flushing mob file a3efb8ce7147f2125466d71154ac42aa/A: closing flushed file at 1731671541293Flushing a3efb8ce7147f2125466d71154ac42aa/A: appending metadata at 1731671541705 (+412 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/A: closing flushed file at 1731671541705Flushing a3efb8ce7147f2125466d71154ac42aa/B: creating writer at 1731671542149 (+444 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/B: appending metadata at 1731671542153 (+4 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/B: closing flushed file at 1731671542153Flushing a3efb8ce7147f2125466d71154ac42aa/C: creating writer at 1731671542565 (+412 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/C: appending metadata at 1731671542568 (+3 ms)Flushing a3efb8ce7147f2125466d71154ac42aa/C: closing flushed file at 1731671542569 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cb56e0e: reopening flushed file at 1731671542978 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37b20408: reopening flushed file at 1731671542983 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0c3362: reopening flushed file at 1731671542987 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a3efb8ce7147f2125466d71154ac42aa in 1706ms, sequenceid=710, compaction requested=true at 1731671542991 (+4 ms)Writing region close event to WAL at 1731671543113 (+122 ms)Running coprocessor post-close hooks at 1731671543117 (+4 ms)Closed at 1731671543117 2024-11-15T11:52:23,119 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,119 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=a3efb8ce7147f2125466d71154ac42aa, regionState=CLOSED 2024-11-15T11:52:23,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:23,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T11:52:23,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure a3efb8ce7147f2125466d71154ac42aa, server=7adf9b3d9d04,44479,1731671471201 in 1.9910 sec 2024-11-15T11:52:23,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T11:52:23,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a3efb8ce7147f2125466d71154ac42aa, UNASSIGN in 2.0000 sec 2024-11-15T11:52:23,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T11:52:23,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0040 sec 2024-11-15T11:52:23,128 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671543127"}]},"ts":"1731671543127"} 2024-11-15T11:52:23,129 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:52:23,129 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:52:23,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0140 sec 2024-11-15T11:52:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T11:52:23,261 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:23,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:52:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,263 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T11:52:23,264 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,267 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,269 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits] 2024-11-15T11:52:23,272 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/059edbd2641d46f8be1caf3031b4489f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/059edbd2641d46f8be1caf3031b4489f 2024-11-15T11:52:23,273 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/2a2adbbd8b7448a7b972874aa6969eb7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/2a2adbbd8b7448a7b972874aa6969eb7 2024-11-15T11:52:23,274 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/508bd51eb5b743749412b6c425677b67 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/A/508bd51eb5b743749412b6c425677b67 2024-11-15T11:52:23,276 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/5ae148f1cc164abebc3c8552d07e549e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/5ae148f1cc164abebc3c8552d07e549e 2024-11-15T11:52:23,277 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/758fb3c0736942ca9247516222e42d00 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/758fb3c0736942ca9247516222e42d00 2024-11-15T11:52:23,278 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe56ee30d8f24e7ab26067685b2393af to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/B/fe56ee30d8f24e7ab26067685b2393af 2024-11-15T11:52:23,281 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2bf5a89039584533b7fd97c3a73cc684 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/2bf5a89039584533b7fd97c3a73cc684 2024-11-15T11:52:23,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8a589d6141c24e35a63c55c62d944a03 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/8a589d6141c24e35a63c55c62d944a03 2024-11-15T11:52:23,283 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/ef09ce7b2f9944079c6874262fe1f61f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/C/ef09ce7b2f9944079c6874262fe1f61f 2024-11-15T11:52:23,285 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits/713.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa/recovered.edits/713.seqid 2024-11-15T11:52:23,286 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,286 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:52:23,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:52:23,287 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T11:52:23,292 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115073cc7f6bdc349208812fdb4edbc1ad5_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115073cc7f6bdc349208812fdb4edbc1ad5_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,293 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150ece1b6c58044d939694481ae5c94af0_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150ece1b6c58044d939694481ae5c94af0_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,294 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111521992a4cb60b4b249df82a08df28faec_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111521992a4cb60b4b249df82a08df28faec_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,295 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115332f12795785403bb5427eb3d26398f9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115332f12795785403bb5427eb3d26398f9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,296 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f80144090424d1bb5ec85f243aaac0a_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f80144090424d1bb5ec85f243aaac0a_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,298 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115451994cf2e754e509e469779f59b5b45_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115451994cf2e754e509e469779f59b5b45_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,299 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115614f928a156344c5a36403ce874988a5_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115614f928a156344c5a36403ce874988a5_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,300 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115709c00b61dd04ab4a5a792eb3a1fdd6e_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115709c00b61dd04ab4a5a792eb3a1fdd6e_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,302 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575ed5e49edbc4868b28ac7b71bed93d6_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111575ed5e49edbc4868b28ac7b71bed93d6_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,303 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158af28f2b90644ec6a36735dae44c4ed9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158af28f2b90644ec6a36735dae44c4ed9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,304 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590cfd550d5d24eb9a159a2fe481edf8c_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590cfd550d5d24eb9a159a2fe481edf8c_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,305 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b943047693b4a528de80cb9a2d2c83a_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159b943047693b4a528de80cb9a2d2c83a_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,305 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a406ba6d604c466fa2c0f257551ef431_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a406ba6d604c466fa2c0f257551ef431_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,307 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a408af4c74974f5caac5f16cf44889c0_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a408af4c74974f5caac5f16cf44889c0_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,308 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a5106552c55c4c61a1b25ccd49a81ee6_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a5106552c55c4c61a1b25ccd49a81ee6_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,312 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a707484e062d43cda01e1acc2ffdf1ef_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a707484e062d43cda01e1acc2ffdf1ef_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,313 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfdd48bb91a0416fb085627946e8e1c9_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bfdd48bb91a0416fb085627946e8e1c9_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,315 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0737d8e4edc4f30bb2484c5e16e2200_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0737d8e4edc4f30bb2484c5e16e2200_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,317 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d5e1e2e809d64ec1a16e4234a7f9a575_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d5e1e2e809d64ec1a16e4234a7f9a575_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,318 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df3394775b37439bb831c4c863ab0924_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df3394775b37439bb831c4c863ab0924_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,320 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e144022b1dac4f1489be1d69bdc03c3d_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e144022b1dac4f1489be1d69bdc03c3d_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,321 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6df7bf833d844209749cf7f8a6fc0b7_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6df7bf833d844209749cf7f8a6fc0b7_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,322 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115efe2227687f74a0e9068576acbb16c5c_a3efb8ce7147f2125466d71154ac42aa to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115efe2227687f74a0e9068576acbb16c5c_a3efb8ce7147f2125466d71154ac42aa 2024-11-15T11:52:23,322 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:52:23,324 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,328 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:52:23,330 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:52:23,331 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,331 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:52:23,331 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671543331"}]},"ts":"9223372036854775807"} 2024-11-15T11:52:23,334 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:52:23,334 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a3efb8ce7147f2125466d71154ac42aa, NAME => 'TestAcidGuarantees,,1731671510799.a3efb8ce7147f2125466d71154ac42aa.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:52:23,334 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:52:23,334 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671543334"}]},"ts":"9223372036854775807"} 2024-11-15T11:52:23,336 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:52:23,337 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-11-15T11:52:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T11:52:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:52:23,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:23,383 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 241) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-160368578_22 at /127.0.0.1:59896 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 324) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10344 (was 10451) 2024-11-15T11:52:23,394 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=10344 2024-11-15T11:52:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:52:23,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:52:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:23,398 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:52:23,398 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:23,398 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-15T11:52:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:23,399 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:52:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742072_1248 (size=1061) 2024-11-15T11:52:23,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:23,808 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:52:23,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742073_1249 (size=53) 2024-11-15T11:52:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 962de2b7258c31da99a4c1635a09f927, disabling compactions & flushes 2024-11-15T11:52:24,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. after waiting 0 ms 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,222 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,222 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 962de2b7258c31da99a4c1635a09f927: Waiting for close lock at 1731671544222Disabling compacts and flushes for region at 1731671544222Disabling writes for close at 1731671544222Writing region close event to WAL at 1731671544222Closed at 1731671544222 2024-11-15T11:52:24,225 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:52:24,225 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671544225"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671544225"}]},"ts":"1731671544225"} 2024-11-15T11:52:24,227 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:52:24,228 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:52:24,229 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671544228"}]},"ts":"1731671544228"} 2024-11-15T11:52:24,230 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:52:24,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, ASSIGN}] 2024-11-15T11:52:24,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, ASSIGN 2024-11-15T11:52:24,233 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:52:24,383 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=962de2b7258c31da99a4c1635a09f927, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:24,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, ASSIGN because future has completed 2024-11-15T11:52:24,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:24,540 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,540 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:52:24,541 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,541 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:52:24,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:24,541 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,541 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,543 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,544 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:24,545 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 962de2b7258c31da99a4c1635a09f927 columnFamilyName A 2024-11-15T11:52:24,545 DEBUG [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:24,545 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(327): Store=962de2b7258c31da99a4c1635a09f927/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:24,546 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,547 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:24,547 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 962de2b7258c31da99a4c1635a09f927 columnFamilyName B 2024-11-15T11:52:24,547 DEBUG [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:24,548 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(327): Store=962de2b7258c31da99a4c1635a09f927/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:24,548 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,550 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:24,550 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 962de2b7258c31da99a4c1635a09f927 columnFamilyName C 2024-11-15T11:52:24,550 DEBUG [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:24,551 INFO [StoreOpener-962de2b7258c31da99a4c1635a09f927-1 {}] regionserver.HStore(327): Store=962de2b7258c31da99a4c1635a09f927/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:24,551 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,551 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,553 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,553 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,555 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,555 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,556 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:52:24,557 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,559 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:52:24,560 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened 962de2b7258c31da99a4c1635a09f927; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63630265, jitterRate=-0.05183516442775726}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:52:24,560 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:24,561 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for 962de2b7258c31da99a4c1635a09f927: Running coprocessor pre-open hook at 1731671544541Writing region info on filesystem at 1731671544541Initializing all the Stores at 1731671544542 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671544542Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671544543 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671544543Cleaning up temporary data from old regions at 1731671544555 (+12 ms)Running coprocessor post-open hooks at 1731671544560 (+5 ms)Region opened successfully at 1731671544561 (+1 ms) 2024-11-15T11:52:24,562 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., pid=67, masterSystemTime=1731671544536 2024-11-15T11:52:24,563 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,564 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:24,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=962de2b7258c31da99a4c1635a09f927, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:24,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:24,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T11:52:24,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 in 182 msec 2024-11-15T11:52:24,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-15T11:52:24,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, ASSIGN in 338 msec 2024-11-15T11:52:24,571 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:52:24,571 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671544571"}]},"ts":"1731671544571"} 2024-11-15T11:52:24,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:52:24,573 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:52:24,574 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-15T11:52:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-15T11:52:25,552 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:25,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22370d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,562 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588d8843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,563 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:52:25,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:52:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4949afc8 2024-11-15T11:52:25,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:52:25,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:52:25,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,569 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67eb2e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,571 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d93b82a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,572 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,572 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce7111b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,573 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7db7a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,575 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150b1fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@275fffbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,578 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5aa414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,579 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c289034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377df873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a867bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6961b01d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13933407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333c0b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8177df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260bcf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416d89e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acf807a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:25,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:25,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@459b9868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:25,600 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:25,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2597f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:25,601 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,601 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,601 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,601 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:25,601 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,602 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,602 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T11:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,603 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,603 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,604 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:25,604 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,608 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,610 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:25,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:52:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:25,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:25,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671545635, max timestamp=1731671545641Num uniques 859; 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671545635, max timestamp=1731671545641Num uniques 859; 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671545635, max timestamp=1731671545641Num uniques 859; 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:25,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:25,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/d22c355bb7f046c595391d9ce464fb2c is 50, key is test_row_0/A:col10/1731671545610/Put/seqid=0 2024-11-15T11:52:25,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671605659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671605660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671605661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671605663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671605664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742074_1250 (size=14341) 2024-11-15T11:52:25,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/d22c355bb7f046c595391d9ce464fb2c 2024-11-15T11:52:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:25,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e6c2bdfc886427a8cf12da4d25e9913 is 50, key is test_row_0/B:col10/1731671545610/Put/seqid=0 2024-11-15T11:52:25,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742075_1251 (size=12001) 2024-11-15T11:52:25,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e6c2bdfc886427a8cf12da4d25e9913 2024-11-15T11:52:25,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:25,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:25,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:25,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/b25d28827908406bbf81d8d30fdd27a3 is 50, key is test_row_0/C:col10/1731671545610/Put/seqid=0 2024-11-15T11:52:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671605770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671605771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671605771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742076_1252 (size=12001) 2024-11-15T11:52:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671605780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671605780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:25,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:25,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671605981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671605981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671605981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671605990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671605990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/b25d28827908406bbf81d8d30fdd27a3 2024-11-15T11:52:26,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/d22c355bb7f046c595391d9ce464fb2c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c 2024-11-15T11:52:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c, entries=200, sequenceid=11, filesize=14.0 K 2024-11-15T11:52:26,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e6c2bdfc886427a8cf12da4d25e9913 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913 2024-11-15T11:52:26,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T11:52:26,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/b25d28827908406bbf81d8d30fdd27a3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3 2024-11-15T11:52:26,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3, entries=150, sequenceid=11, filesize=11.7 K 2024-11-15T11:52:26,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 962de2b7258c31da99a4c1635a09f927 in 606ms, sequenceid=11, compaction requested=false 2024-11-15T11:52:26,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:26,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:26,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7c7d4eb6d2e346788599b08b17f1f69b is 50, key is test_row_0/A:col10/1731671545660/Put/seqid=0 2024-11-15T11:52:26,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671606300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671606301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742077_1253 (size=14341) 2024-11-15T11:52:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671606302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7c7d4eb6d2e346788599b08b17f1f69b 2024-11-15T11:52:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671606307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671606307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/a988054f6bed4b75ae9b045212bd83b3 is 50, key is test_row_0/B:col10/1731671545660/Put/seqid=0 2024-11-15T11:52:26,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742078_1254 (size=12001) 2024-11-15T11:52:26,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/a988054f6bed4b75ae9b045212bd83b3 2024-11-15T11:52:26,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/e7eb0a2c12ad4f1d9846f833bb725b59 is 50, key is test_row_0/C:col10/1731671545660/Put/seqid=0 2024-11-15T11:52:26,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742079_1255 (size=12001) 2024-11-15T11:52:26,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/e7eb0a2c12ad4f1d9846f833bb725b59 2024-11-15T11:52:26,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7c7d4eb6d2e346788599b08b17f1f69b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b 2024-11-15T11:52:26,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b, entries=200, sequenceid=48, filesize=14.0 K 2024-11-15T11:52:26,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/a988054f6bed4b75ae9b045212bd83b3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3 2024-11-15T11:52:26,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T11:52:26,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/e7eb0a2c12ad4f1d9846f833bb725b59 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59 2024-11-15T11:52:26,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59, entries=150, sequenceid=48, filesize=11.7 K 2024-11-15T11:52:26,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 962de2b7258c31da99a4c1635a09f927 in 82ms, sequenceid=48, compaction requested=false 2024-11-15T11:52:26,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:26,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:52:26,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:26,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:26,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8d97097a1ede4a91b1644f87ef247832 is 50, key is test_row_0/A:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:26,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742080_1256 (size=14341) 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671546439, max timestamp=1731671546454Num uniques 859; 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671546439, max timestamp=1731671546454Num uniques 859; 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671546439, max timestamp=1731671546454Num uniques 859; 2024-11-15T11:52:26,457 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671606476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671606477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671606479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671606483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671606591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671606591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671606591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671606591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671606800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671606800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671606800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671606811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671606811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:26,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8d97097a1ede4a91b1644f87ef247832 2024-11-15T11:52:26,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/45dc944f30c04b51a88565cd31fbf06e is 50, key is test_row_0/B:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:26,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742081_1257 (size=12001) 2024-11-15T11:52:26,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/45dc944f30c04b51a88565cd31fbf06e 2024-11-15T11:52:26,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2ebb7eb3231540ef937d5115c82215bd is 50, key is test_row_0/C:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:26,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742082_1258 (size=12001) 2024-11-15T11:52:26,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:26,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671607110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671607110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671607110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671607121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2ebb7eb3231540ef937d5115c82215bd 2024-11-15T11:52:27,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8d97097a1ede4a91b1644f87ef247832 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832 2024-11-15T11:52:27,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832, entries=200, sequenceid=59, filesize=14.0 K 2024-11-15T11:52:27,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/45dc944f30c04b51a88565cd31fbf06e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e 2024-11-15T11:52:27,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:27,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T11:52:27,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2ebb7eb3231540ef937d5115c82215bd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd 2024-11-15T11:52:27,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd, entries=150, sequenceid=59, filesize=11.7 K 2024-11-15T11:52:27,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 962de2b7258c31da99a4c1635a09f927 in 907ms, sequenceid=59, compaction requested=true 2024-11-15T11:52:27,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:27,323 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:52:27,323 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:27,324 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:27,324 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:27,324 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:27,324 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:27,324 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,324 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,324 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=42.0 K 2024-11-15T11:52:27,324 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.2 K 2024-11-15T11:52:27,325 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e6c2bdfc886427a8cf12da4d25e9913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731671545610 2024-11-15T11:52:27,325 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d22c355bb7f046c595391d9ce464fb2c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731671545610 2024-11-15T11:52:27,325 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a988054f6bed4b75ae9b045212bd83b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731671545658 2024-11-15T11:52:27,325 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7d4eb6d2e346788599b08b17f1f69b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731671545658 2024-11-15T11:52:27,325 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45dc944f30c04b51a88565cd31fbf06e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546305 2024-11-15T11:52:27,326 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d97097a1ede4a91b1644f87ef247832, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546298 2024-11-15T11:52:27,349 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:27,350 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/f6f5ae9701224ed5970181c3b954aa41 is 50, key is test_row_0/A:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:27,357 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:27,358 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/8e373623d01042ca802a79ade0d38c6c is 50, key is test_row_0/B:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742083_1259 (size=12104) 2024-11-15T11:52:27,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742084_1260 (size=12104) 2024-11-15T11:52:27,401 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/8e373623d01042ca802a79ade0d38c6c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8e373623d01042ca802a79ade0d38c6c 2024-11-15T11:52:27,408 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 8e373623d01042ca802a79ade0d38c6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:27,408 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:27,408 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671547323; duration=0sec 2024-11-15T11:52:27,408 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:27,408 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:27,408 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:27,410 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:27,411 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:27,411 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,411 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.2 K 2024-11-15T11:52:27,411 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25d28827908406bbf81d8d30fdd27a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731671545610 2024-11-15T11:52:27,412 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7eb0a2c12ad4f1d9846f833bb725b59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731671545658 2024-11-15T11:52:27,412 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ebb7eb3231540ef937d5115c82215bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546305 2024-11-15T11:52:27,433 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:27,433 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/aa58980beb33489a87998fdb40fbf831 is 50, key is test_row_0/C:col10/1731671546305/Put/seqid=0 2024-11-15T11:52:27,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T11:52:27,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:27,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:27,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e3fc3ef2c8044659abc565efbdc948d is 50, key is test_row_0/A:col10/1731671546475/Put/seqid=0 2024-11-15T11:52:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742085_1261 (size=12104) 2024-11-15T11:52:27,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742086_1262 (size=12001) 2024-11-15T11:52:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:27,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671607632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671607633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671607634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671607640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671607751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671607751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671607751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671607751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:27,796 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/f6f5ae9701224ed5970181c3b954aa41 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/f6f5ae9701224ed5970181c3b954aa41 2024-11-15T11:52:27,799 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into f6f5ae9701224ed5970181c3b954aa41(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:27,799 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:27,799 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671547323; duration=0sec 2024-11-15T11:52:27,799 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:27,799 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:27,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671607820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,823 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,889 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/aa58980beb33489a87998fdb40fbf831 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa58980beb33489a87998fdb40fbf831 2024-11-15T11:52:27,894 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e3fc3ef2c8044659abc565efbdc948d 2024-11-15T11:52:27,895 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into aa58980beb33489a87998fdb40fbf831(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:27,895 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:27,895 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671547323; duration=0sec 2024-11-15T11:52:27,895 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:27,895 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:27,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/38c49da9555d4b9f8e17c0d1c6f6e250 is 50, key is test_row_0/B:col10/1731671546475/Put/seqid=0 2024-11-15T11:52:27,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742087_1263 (size=12001) 2024-11-15T11:52:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671607960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671607960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671607960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671607960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671608271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671608271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671608271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671608271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,306 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/38c49da9555d4b9f8e17c0d1c6f6e250 2024-11-15T11:52:28,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/df9b16503f1c4d01917a4d2b0ccfeb2a is 50, key is test_row_0/C:col10/1731671546475/Put/seqid=0 2024-11-15T11:52:28,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742088_1264 (size=12001) 2024-11-15T11:52:28,369 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/df9b16503f1c4d01917a4d2b0ccfeb2a 2024-11-15T11:52:28,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e3fc3ef2c8044659abc565efbdc948d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d 2024-11-15T11:52:28,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T11:52:28,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/38c49da9555d4b9f8e17c0d1c6f6e250 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250 2024-11-15T11:52:28,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T11:52:28,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/df9b16503f1c4d01917a4d2b0ccfeb2a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a 2024-11-15T11:52:28,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T11:52:28,396 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 962de2b7258c31da99a4c1635a09f927 in 949ms, sequenceid=95, compaction requested=false 2024-11-15T11:52:28,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:28,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:28,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T11:52:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T11:52:28,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T11:52:28,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7930 sec 2024-11-15T11:52:28,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.7980 sec 2024-11-15T11:52:28,648 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:52:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:28,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:28,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e699dc8bde5420eb1f39c3fbc95e0fe is 50, key is test_row_0/A:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:28,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742089_1265 (size=12001) 2024-11-15T11:52:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671608805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671608805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671608806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671608809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671608920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671608920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671608920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:28,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671608920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:28,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671609130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671609130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671609130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671609130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e699dc8bde5420eb1f39c3fbc95e0fe 2024-11-15T11:52:29,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/467b91b339094f9c8de8b0c6c7f5009e is 50, key is test_row_0/B:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:29,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742090_1266 (size=12001) 2024-11-15T11:52:29,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/467b91b339094f9c8de8b0c6c7f5009e 2024-11-15T11:52:29,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/8f1bf0d582dd445ea6c4426d54718b2b is 50, key is test_row_0/C:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742091_1267 (size=12001) 2024-11-15T11:52:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671609441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/8f1bf0d582dd445ea6c4426d54718b2b 2024-11-15T11:52:29,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/1e699dc8bde5420eb1f39c3fbc95e0fe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe 2024-11-15T11:52:29,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T11:52:29,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/467b91b339094f9c8de8b0c6c7f5009e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e 2024-11-15T11:52:29,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T11:52:29,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/8f1bf0d582dd445ea6c4426d54718b2b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b 2024-11-15T11:52:29,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T11:52:29,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 962de2b7258c31da99a4c1635a09f927 in 917ms, sequenceid=110, compaction requested=true 2024-11-15T11:52:29,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:29,700 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:29,701 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:29,701 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:29,701 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:29,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:29,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:29,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:29,702 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:29,702 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/f6f5ae9701224ed5970181c3b954aa41, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.3 K 2024-11-15T11:52:29,702 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6f5ae9701224ed5970181c3b954aa41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546305 2024-11-15T11:52:29,703 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:29,703 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e3fc3ef2c8044659abc565efbdc948d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671546466 2024-11-15T11:52:29,703 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:29,703 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:29,703 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8e373623d01042ca802a79ade0d38c6c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.3 K 2024-11-15T11:52:29,703 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e699dc8bde5420eb1f39c3fbc95e0fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:29,703 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e373623d01042ca802a79ade0d38c6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546305 2024-11-15T11:52:29,704 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c49da9555d4b9f8e17c0d1c6f6e250, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671546466 2024-11-15T11:52:29,704 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 467b91b339094f9c8de8b0c6c7f5009e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:29,715 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:29,717 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/6a8b33a769184e54b6709431ca44e40a is 50, key is test_row_0/A:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:29,730 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:29,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/e52861b7f780435eb839a58c84456916 is 50, key is test_row_0/B:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:29,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742093_1269 (size=12207) 2024-11-15T11:52:29,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742092_1268 (size=12207) 2024-11-15T11:52:29,750 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/6a8b33a769184e54b6709431ca44e40a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a8b33a769184e54b6709431ca44e40a 2024-11-15T11:52:29,757 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 6a8b33a769184e54b6709431ca44e40a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:29,757 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:29,757 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671549700; duration=0sec 2024-11-15T11:52:29,758 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:29,758 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:29,758 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:29,759 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:29,759 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:29,759 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:29,759 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa58980beb33489a87998fdb40fbf831, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.3 K 2024-11-15T11:52:29,760 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa58980beb33489a87998fdb40fbf831, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731671546305 2024-11-15T11:52:29,760 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting df9b16503f1c4d01917a4d2b0ccfeb2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671546466 2024-11-15T11:52:29,760 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1bf0d582dd445ea6c4426d54718b2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T11:52:29,770 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:29,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:29,772 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:29,773 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/05cf72936176495099500cee1c1831ca is 50, key is test_row_0/C:col10/1731671548783/Put/seqid=0 2024-11-15T11:52:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T11:52:29,776 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:29,777 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:29,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T11:52:29,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742094_1270 (size=12207) 2024-11-15T11:52:29,795 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/05cf72936176495099500cee1c1831ca as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/05cf72936176495099500cee1c1831ca 2024-11-15T11:52:29,801 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 05cf72936176495099500cee1c1831ca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:29,801 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:29,801 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671549702; duration=0sec 2024-11-15T11:52:29,801 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:29,801 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:29,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:29,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/60ff7713d79944bdb26e0c3539ee7be6 is 50, key is test_row_0/A:col10/1731671548802/Put/seqid=0 2024-11-15T11:52:29,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742095_1271 (size=14391) 2024-11-15T11:52:29,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/60ff7713d79944bdb26e0c3539ee7be6 2024-11-15T11:52:29,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/58d4ed6995724c27a7983b1f5421b25e is 50, key is test_row_0/B:col10/1731671548802/Put/seqid=0 2024-11-15T11:52:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T11:52:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742096_1272 (size=12051) 2024-11-15T11:52:29,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/58d4ed6995724c27a7983b1f5421b25e 2024-11-15T11:52:29,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671609904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/333b535311684ad7877bf6ba11ab47c0 is 50, key is test_row_0/C:col10/1731671548802/Put/seqid=0 2024-11-15T11:52:29,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T11:52:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:29,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:29,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:29,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742097_1273 (size=12051) 2024-11-15T11:52:29,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671609950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671609950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671609950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:29,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671609950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:29,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671610010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T11:52:30,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:30,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T11:52:30,149 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/e52861b7f780435eb839a58c84456916 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e52861b7f780435eb839a58c84456916 2024-11-15T11:52:30,155 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into e52861b7f780435eb839a58c84456916(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:30,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:30,155 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671549701; duration=0sec 2024-11-15T11:52:30,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:30,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671610220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T11:52:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:30,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/333b535311684ad7877bf6ba11ab47c0 2024-11-15T11:52:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/60ff7713d79944bdb26e0c3539ee7be6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6 2024-11-15T11:52:30,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6, entries=200, sequenceid=132, filesize=14.1 K 2024-11-15T11:52:30,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/58d4ed6995724c27a7983b1f5421b25e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e 2024-11-15T11:52:30,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T11:52:30,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/333b535311684ad7877bf6ba11ab47c0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0 2024-11-15T11:52:30,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T11:52:30,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 962de2b7258c31da99a4c1635a09f927 in 509ms, sequenceid=132, compaction requested=false 2024-11-15T11:52:30,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:30,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T11:52:30,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,392 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:30,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T11:52:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/9ac323ba0be8478fbacc23f1a532daeb is 50, key is test_row_0/A:col10/1731671549874/Put/seqid=0 2024-11-15T11:52:30,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742098_1274 (size=12151) 2024-11-15T11:52:30,434 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/9ac323ba0be8478fbacc23f1a532daeb 2024-11-15T11:52:30,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/1573143ba95a46449e24c6f2723990cf is 50, key is test_row_0/B:col10/1731671549874/Put/seqid=0 2024-11-15T11:52:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742099_1275 (size=12151) 2024-11-15T11:52:30,499 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/1573143ba95a46449e24c6f2723990cf 2024-11-15T11:52:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/080eb44123d04c2491f93698352d7b77 is 50, key is test_row_0/C:col10/1731671549874/Put/seqid=0 2024-11-15T11:52:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:30,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:30,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742100_1276 (size=12151) 2024-11-15T11:52:30,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/080eb44123d04c2491f93698352d7b77 2024-11-15T11:52:30,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/9ac323ba0be8478fbacc23f1a532daeb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb 2024-11-15T11:52:30,566 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T11:52:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/1573143ba95a46449e24c6f2723990cf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf 2024-11-15T11:52:30,574 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T11:52:30,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/080eb44123d04c2491f93698352d7b77 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77 2024-11-15T11:52:30,582 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T11:52:30,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=60.38 KB/61830 for 962de2b7258c31da99a4c1635a09f927 in 191ms, sequenceid=145, compaction requested=true 2024-11-15T11:52:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T11:52:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T11:52:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:30,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:30,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:30,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T11:52:30,587 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 808 msec 2024-11-15T11:52:30,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 817 msec 2024-11-15T11:52:30,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8a45bd099714405fad107d23ef211ff6 is 50, key is test_row_0/A:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:30,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742101_1277 (size=14541) 2024-11-15T11:52:30,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8a45bd099714405fad107d23ef211ff6 2024-11-15T11:52:30,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/23e47c57e8a04f029b5ee34f622e10c2 is 50, key is test_row_0/B:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742102_1278 (size=12151) 2024-11-15T11:52:30,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/23e47c57e8a04f029b5ee34f622e10c2 2024-11-15T11:52:30,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/30eb126323c84f888353bf61cc892635 is 50, key is test_row_0/C:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:30,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742103_1279 (size=12151) 2024-11-15T11:52:30,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671610692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671610800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T11:52:30,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:30,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T11:52:30,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T11:52:30,913 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:30,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:30,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671610961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671610961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,965 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,965 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671610970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671610970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:30,974 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:30,975 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:31,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671611011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T11:52:31,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/30eb126323c84f888353bf61cc892635 2024-11-15T11:52:31,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/8a45bd099714405fad107d23ef211ff6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6 2024-11-15T11:52:31,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6, entries=200, sequenceid=158, filesize=14.2 K 2024-11-15T11:52:31,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/23e47c57e8a04f029b5ee34f622e10c2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2 2024-11-15T11:52:31,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T11:52:31,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/30eb126323c84f888353bf61cc892635 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635 2024-11-15T11:52:31,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T11:52:31,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 962de2b7258c31da99a4c1635a09f927 in 471ms, sequenceid=158, compaction requested=true 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:31,057 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:52:31,058 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:31,058 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:31,059 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:31,059 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:31,059 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a8b33a769184e54b6709431ca44e40a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=52.0 K 2024-11-15T11:52:31,059 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:31,059 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8b33a769184e54b6709431ca44e40a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:31,059 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:31,059 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:31,059 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e52861b7f780435eb839a58c84456916, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=47.4 K 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ff7713d79944bdb26e0c3539ee7be6, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731671548802 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e52861b7f780435eb839a58c84456916, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d4ed6995724c27a7983b1f5421b25e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731671548802 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac323ba0be8478fbacc23f1a532daeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671549874 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1573143ba95a46449e24c6f2723990cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671549874 2024-11-15T11:52:31,060 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a45bd099714405fad107d23ef211ff6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:31,061 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e47c57e8a04f029b5ee34f622e10c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:31,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:31,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/e66eb60c72aa40d58374f6fc934c2182 is 50, key is test_row_0/A:col10/1731671550690/Put/seqid=0 2024-11-15T11:52:31,074 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:31,075 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/1f0a5ed122dc4d7caddbb1d683875a40 is 50, key is test_row_0/B:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:31,080 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:31,081 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/260d952ca1e24834968dbf2ca773744a is 50, key is test_row_0/A:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:31,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742104_1280 (size=12151) 2024-11-15T11:52:31,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/e66eb60c72aa40d58374f6fc934c2182 2024-11-15T11:52:31,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742105_1281 (size=12493) 2024-11-15T11:52:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/21428fed0c524921b2c6e312cb24741b is 50, key is test_row_0/B:col10/1731671550690/Put/seqid=0 2024-11-15T11:52:31,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742106_1282 (size=12493) 2024-11-15T11:52:31,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742107_1283 (size=12151) 2024-11-15T11:52:31,137 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/21428fed0c524921b2c6e312cb24741b 2024-11-15T11:52:31,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/1afb2ed341844c1d910de39ffcdc105e is 50, key is test_row_0/C:col10/1731671550690/Put/seqid=0 2024-11-15T11:52:31,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742108_1284 (size=12151) 2024-11-15T11:52:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T11:52:31,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:31,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:31,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:31,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671611379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:31,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671611503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:31,517 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/1f0a5ed122dc4d7caddbb1d683875a40 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1f0a5ed122dc4d7caddbb1d683875a40 2024-11-15T11:52:31,530 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 1f0a5ed122dc4d7caddbb1d683875a40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:31,530 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:31,530 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=12, startTime=1731671551057; duration=0sec 2024-11-15T11:52:31,531 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:31,531 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:31,531 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:31,533 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/260d952ca1e24834968dbf2ca773744a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/260d952ca1e24834968dbf2ca773744a 2024-11-15T11:52:31,534 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:31,534 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:31,534 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:31,534 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/05cf72936176495099500cee1c1831ca, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=47.4 K 2024-11-15T11:52:31,537 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 05cf72936176495099500cee1c1831ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671547632 2024-11-15T11:52:31,537 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 333b535311684ad7877bf6ba11ab47c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731671548802 2024-11-15T11:52:31,539 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 080eb44123d04c2491f93698352d7b77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671549874 2024-11-15T11:52:31,540 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 260d952ca1e24834968dbf2ca773744a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:31,540 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:31,540 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=12, startTime=1731671551057; duration=0sec 2024-11-15T11:52:31,540 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:31,540 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T11:52:31,540 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 30eb126323c84f888353bf61cc892635, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:31,578 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:31,578 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/774f0264a8bf4795bbcf2773ec12ce31 is 50, key is test_row_0/C:col10/1731671550536/Put/seqid=0 2024-11-15T11:52:31,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/1afb2ed341844c1d910de39ffcdc105e 2024-11-15T11:52:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/e66eb60c72aa40d58374f6fc934c2182 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182 2024-11-15T11:52:31,594 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:52:31,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/21428fed0c524921b2c6e312cb24741b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b 2024-11-15T11:52:31,600 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:52:31,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/1afb2ed341844c1d910de39ffcdc105e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e 2024-11-15T11:52:31,606 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e, entries=150, sequenceid=177, filesize=11.9 K 2024-11-15T11:52:31,609 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 962de2b7258c31da99a4c1635a09f927 in 543ms, sequenceid=177, compaction requested=false 2024-11-15T11:52:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T11:52:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T11:52:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742109_1285 (size=12493) 2024-11-15T11:52:31,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T11:52:31,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 698 msec 2024-11-15T11:52:31,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 704 msec 2024-11-15T11:52:31,622 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/774f0264a8bf4795bbcf2773ec12ce31 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/774f0264a8bf4795bbcf2773ec12ce31 2024-11-15T11:52:31,632 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 774f0264a8bf4795bbcf2773ec12ce31(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:31,632 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:31,632 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=12, startTime=1731671551057; duration=0sec 2024-11-15T11:52:31,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:31,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:31,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:31,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0d73bb0324cc412cba37aff04d3a816b is 50, key is test_row_0/A:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:31,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742110_1286 (size=12151) 2024-11-15T11:52:31,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0d73bb0324cc412cba37aff04d3a816b 2024-11-15T11:52:31,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/40bb148e95cf45f9a171e4619aaae620 is 50, key is test_row_0/B:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671611826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:31,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742111_1287 (size=12151) 2024-11-15T11:52:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671611940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:31,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T11:52:32,050 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:32,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T11:52:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:32,053 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:32,054 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:32,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:32,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671612150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:32,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:32,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T11:52:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/40bb148e95cf45f9a171e4619aaae620 2024-11-15T11:52:32,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/ba3d3e66fce84fc48b6a994d35f53334 is 50, key is test_row_0/C:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:32,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742112_1288 (size=12151) 2024-11-15T11:52:32,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T11:52:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671612460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:32,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:32,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T11:52:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T11:52:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:32,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:32,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/ba3d3e66fce84fc48b6a994d35f53334 2024-11-15T11:52:32,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0d73bb0324cc412cba37aff04d3a816b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b 2024-11-15T11:52:32,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T11:52:32,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/40bb148e95cf45f9a171e4619aaae620 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620 2024-11-15T11:52:32,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T11:52:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/ba3d3e66fce84fc48b6a994d35f53334 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334 2024-11-15T11:52:32,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T11:52:32,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 962de2b7258c31da99a4c1635a09f927 in 1000ms, sequenceid=194, compaction requested=true 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:32,723 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:32,723 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:32,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:32,727 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:32,727 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:32,727 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:32,727 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:32,727 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,727 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,728 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/260d952ca1e24834968dbf2ca773744a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.9 K 2024-11-15T11:52:32,728 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1f0a5ed122dc4d7caddbb1d683875a40, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.9 K 2024-11-15T11:52:32,728 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f0a5ed122dc4d7caddbb1d683875a40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:32,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260d952ca1e24834968dbf2ca773744a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:32,728 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 21428fed0c524921b2c6e312cb24741b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671550677 2024-11-15T11:52:32,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66eb60c72aa40d58374f6fc934c2182, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671550677 2024-11-15T11:52:32,729 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 40bb148e95cf45f9a171e4619aaae620, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:32,729 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d73bb0324cc412cba37aff04d3a816b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:32,747 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:32,748 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/c7070ceac1c34c7b8182cae7c120a6f1 is 50, key is test_row_0/A:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:32,761 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:32,761 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/4ee27aceee964651a3a75f927751a08d is 50, key is test_row_0/B:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:32,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742113_1289 (size=12595) 2024-11-15T11:52:32,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742114_1290 (size=12595) 2024-11-15T11:52:32,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T11:52:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:32,821 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-15T11:52:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:32,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/b126d9c2b99d469286c461eb54afac65 is 50, key is test_row_0/A:col10/1731671551797/Put/seqid=0 2024-11-15T11:52:32,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742115_1291 (size=12151) 2024-11-15T11:52:32,842 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/b126d9c2b99d469286c461eb54afac65 2024-11-15T11:52:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e063660cf1941e9bc29ca68908747f9 is 50, key is test_row_0/B:col10/1731671551797/Put/seqid=0 2024-11-15T11:52:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742116_1292 (size=12151) 2024-11-15T11:52:32,881 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e063660cf1941e9bc29ca68908747f9 2024-11-15T11:52:32,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5fc341210d9f4aa3b8c5642e662bb2b9 is 50, key is test_row_0/C:col10/1731671551797/Put/seqid=0 2024-11-15T11:52:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742117_1293 (size=12151) 2024-11-15T11:52:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:32,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:33,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,189 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/c7070ceac1c34c7b8182cae7c120a6f1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/c7070ceac1c34c7b8182cae7c120a6f1 2024-11-15T11:52:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:33,200 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into c7070ceac1c34c7b8182cae7c120a6f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:33,200 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,200 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671552723; duration=0sec 2024-11-15T11:52:33,200 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:33,200 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:33,200 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:33,202 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:33,202 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:33,202 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:33,202 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/774f0264a8bf4795bbcf2773ec12ce31, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=35.9 K 2024-11-15T11:52:33,203 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774f0264a8bf4795bbcf2773ec12ce31, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731671550536 2024-11-15T11:52:33,204 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1afb2ed341844c1d910de39ffcdc105e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731671550677 2024-11-15T11:52:33,204 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba3d3e66fce84fc48b6a994d35f53334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:33,206 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/4ee27aceee964651a3a75f927751a08d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4ee27aceee964651a3a75f927751a08d 2024-11-15T11:52:33,213 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 4ee27aceee964651a3a75f927751a08d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:33,213 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,213 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671552723; duration=0sec 2024-11-15T11:52:33,213 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:33,213 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:33,216 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:33,216 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/7278660eb6124a8fbfbce257593678a8 is 50, key is test_row_0/C:col10/1731671551377/Put/seqid=0 2024-11-15T11:52:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742118_1294 (size=12595) 2024-11-15T11:52:33,317 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5fc341210d9f4aa3b8c5642e662bb2b9 2024-11-15T11:52:33,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/b126d9c2b99d469286c461eb54afac65 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65 2024-11-15T11:52:33,328 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T11:52:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e063660cf1941e9bc29ca68908747f9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9 2024-11-15T11:52:33,344 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T11:52:33,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5fc341210d9f4aa3b8c5642e662bb2b9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9 2024-11-15T11:52:33,350 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T11:52:33,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 962de2b7258c31da99a4c1635a09f927 in 530ms, sequenceid=212, compaction requested=false 2024-11-15T11:52:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T11:52:33,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T11:52:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:33,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:33,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T11:52:33,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3010 sec 2024-11-15T11:52:33,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3090 sec 2024-11-15T11:52:33,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/39165bc4e01d43609f3e585e4857a703 is 50, key is test_row_0/A:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742119_1295 (size=12151) 2024-11-15T11:52:33,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/39165bc4e01d43609f3e585e4857a703 2024-11-15T11:52:33,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/e6c6036c94c24a36b2ff8e7590cafd26 is 50, key is test_row_0/B:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742120_1296 (size=12151) 2024-11-15T11:52:33,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/e6c6036c94c24a36b2ff8e7590cafd26 2024-11-15T11:52:33,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c9a1d992a69a4d919623ac1fd128e872 is 50, key is test_row_0/C:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742121_1297 (size=12151) 2024-11-15T11:52:33,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c9a1d992a69a4d919623ac1fd128e872 2024-11-15T11:52:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/39165bc4e01d43609f3e585e4857a703 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703 2024-11-15T11:52:33,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T11:52:33,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/e6c6036c94c24a36b2ff8e7590cafd26 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26 2024-11-15T11:52:33,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T11:52:33,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c9a1d992a69a4d919623ac1fd128e872 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872 2024-11-15T11:52:33,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872, entries=150, sequenceid=229, filesize=11.9 K 2024-11-15T11:52:33,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 962de2b7258c31da99a4c1635a09f927 in 189ms, sequenceid=229, compaction requested=true 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:33,546 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:52:33,548 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:33,548 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:33,548 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:33,548 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/c7070ceac1c34c7b8182cae7c120a6f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.0 K 2024-11-15T11:52:33,548 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c7070ceac1c34c7b8182cae7c120a6f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:33,549 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b126d9c2b99d469286c461eb54afac65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731671551797 2024-11-15T11:52:33,549 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 39165bc4e01d43609f3e585e4857a703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:33,577 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:33,577 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0eb3ed95d2b043f3ae415b92cf1b2df2 is 50, key is test_row_0/A:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742122_1298 (size=12697) 2024-11-15T11:52:33,640 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/7278660eb6124a8fbfbce257593678a8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/7278660eb6124a8fbfbce257593678a8 2024-11-15T11:52:33,647 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 7278660eb6124a8fbfbce257593678a8(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:33,647 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,647 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671552723; duration=0sec 2024-11-15T11:52:33,647 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T11:52:33,647 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:33,647 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:33,647 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:33,649 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:33,649 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:33,649 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:33,649 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4ee27aceee964651a3a75f927751a08d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.0 K 2024-11-15T11:52:33,650 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ee27aceee964651a3a75f927751a08d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:33,650 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e063660cf1941e9bc29ca68908747f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731671551797 2024-11-15T11:52:33,650 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c6036c94c24a36b2ff8e7590cafd26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:33,679 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:33,679 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/96c98667238c4f3fa796c6aabd13a513 is 50, key is test_row_0/B:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:52:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:33,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:33,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:33,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:33,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:33,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742123_1299 (size=12697) 2024-11-15T11:52:33,730 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/96c98667238c4f3fa796c6aabd13a513 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/96c98667238c4f3fa796c6aabd13a513 2024-11-15T11:52:33,736 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 96c98667238c4f3fa796c6aabd13a513(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:33,737 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/ddde346d69cb47ffba81a22a20a2d602 is 50, key is test_row_0/A:col10/1731671553374/Put/seqid=0 2024-11-15T11:52:33,737 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671553546; duration=0sec 2024-11-15T11:52:33,737 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:33,737 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:33,737 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:33,739 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:33,739 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:33,739 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:33,739 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/7278660eb6124a8fbfbce257593678a8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.0 K 2024-11-15T11:52:33,740 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7278660eb6124a8fbfbce257593678a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731671551371 2024-11-15T11:52:33,740 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc341210d9f4aa3b8c5642e662bb2b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731671551797 2024-11-15T11:52:33,741 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a1d992a69a4d919623ac1fd128e872, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:33,765 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:33,765 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/f040ffb1c38b4554a2be18e44d9b1313 is 50, key is test_row_0/C:col10/1731671553032/Put/seqid=0 2024-11-15T11:52:33,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742124_1300 (size=12151) 2024-11-15T11:52:33,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/ddde346d69cb47ffba81a22a20a2d602 2024-11-15T11:52:33,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/71daf825990243a3a6ed035bab8ddc7f is 50, key is test_row_0/B:col10/1731671553374/Put/seqid=0 2024-11-15T11:52:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742125_1301 (size=12697) 2024-11-15T11:52:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742126_1302 (size=12151) 2024-11-15T11:52:33,838 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/f040ffb1c38b4554a2be18e44d9b1313 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/f040ffb1c38b4554a2be18e44d9b1313 2024-11-15T11:52:33,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/71daf825990243a3a6ed035bab8ddc7f 2024-11-15T11:52:33,848 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into f040ffb1c38b4554a2be18e44d9b1313(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:33,848 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:33,848 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671553546; duration=0sec 2024-11-15T11:52:33,848 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:33,848 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:33,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c1915ee5c7ad45a6b71a8f32b1a17632 is 50, key is test_row_0/C:col10/1731671553374/Put/seqid=0 2024-11-15T11:52:33,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742127_1303 (size=12151) 2024-11-15T11:52:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671613910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671613910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671613911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671613911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671613911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,035 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0eb3ed95d2b043f3ae415b92cf1b2df2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0eb3ed95d2b043f3ae415b92cf1b2df2 2024-11-15T11:52:34,042 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 0eb3ed95d2b043f3ae415b92cf1b2df2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:34,042 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:34,042 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671553546; duration=0sec 2024-11-15T11:52:34,043 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:34,043 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671614121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671614121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671614121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671614122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671614122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T11:52:34,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:34,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T11:52:34,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:34,205 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:34,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T11:52:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T11:52:34,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c1915ee5c7ad45a6b71a8f32b1a17632 2024-11-15T11:52:34,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/ddde346d69cb47ffba81a22a20a2d602 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602 2024-11-15T11:52:34,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602, entries=150, sequenceid=250, filesize=11.9 K 2024-11-15T11:52:34,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/71daf825990243a3a6ed035bab8ddc7f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f 2024-11-15T11:52:34,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-15T11:52:34,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c1915ee5c7ad45a6b71a8f32b1a17632 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632 2024-11-15T11:52:34,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632, entries=150, sequenceid=250, filesize=11.9 K 2024-11-15T11:52:34,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for 962de2b7258c31da99a4c1635a09f927 in 615ms, sequenceid=250, compaction requested=false 2024-11-15T11:52:34,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:34,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T11:52:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:34,359 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T11:52:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/4cac80767135494aaf506a54d982aff3 is 50, key is test_row_0/A:col10/1731671553738/Put/seqid=0 2024-11-15T11:52:34,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742128_1304 (size=12301) 2024-11-15T11:52:34,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/4cac80767135494aaf506a54d982aff3 2024-11-15T11:52:34,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e5195ba4c79441795ad2272e1d5701a is 50, key is test_row_0/B:col10/1731671553738/Put/seqid=0 2024-11-15T11:52:34,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742129_1305 (size=12301) 2024-11-15T11:52:34,400 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e5195ba4c79441795ad2272e1d5701a 2024-11-15T11:52:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/3d38c2e748e64653aab252576caf3811 is 50, key is test_row_0/C:col10/1731671553738/Put/seqid=0 2024-11-15T11:52:34,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742130_1306 (size=12301) 2024-11-15T11:52:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671614450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671614451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671614455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671614456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671614456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T11:52:34,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671614560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671614561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671614571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671614571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671614571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671614771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671614771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671614780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671614780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671614781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:34,825 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/3d38c2e748e64653aab252576caf3811 2024-11-15T11:52:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T11:52:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/4cac80767135494aaf506a54d982aff3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3 2024-11-15T11:52:34,849 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T11:52:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/0e5195ba4c79441795ad2272e1d5701a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a 2024-11-15T11:52:34,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T11:52:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/3d38c2e748e64653aab252576caf3811 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811 2024-11-15T11:52:34,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811, entries=150, sequenceid=267, filesize=12.0 K 2024-11-15T11:52:34,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 962de2b7258c31da99a4c1635a09f927 in 503ms, sequenceid=267, compaction requested=true 2024-11-15T11:52:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:34,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T11:52:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T11:52:34,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T11:52:34,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-11-15T11:52:34,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 665 msec 2024-11-15T11:52:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:35,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:35,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:35,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/03b78c73c05f40d2adff625a7324a8ea is 50, key is test_row_0/A:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:35,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742131_1307 (size=12301) 2024-11-15T11:52:35,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671615102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671615103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671615104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671615105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671615106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671615211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671615211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671615220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671615220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671615220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T11:52:35,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:35,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T11:52:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:35,354 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:35,356 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:35,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671615420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671615420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671615430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671615430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671615431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:35,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/03b78c73c05f40d2adff625a7324a8ea 2024-11-15T11:52:35,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/07c69275394c41328a7daef88ec0cd7d is 50, key is test_row_0/B:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:35,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742132_1308 (size=12301) 2024-11-15T11:52:35,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:35,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:35,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671615730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671615730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671615741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671615741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671615741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:35,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:35,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/07c69275394c41328a7daef88ec0cd7d 2024-11-15T11:52:35,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/994fa96d3bd54ab3be7652d4660910d0 is 50, key is test_row_0/C:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:35,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742133_1309 (size=12301) 2024-11-15T11:52:35,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:35,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:36,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671616241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671616241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:36,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671616250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671616250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671616250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:36,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:36,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/994fa96d3bd54ab3be7652d4660910d0 2024-11-15T11:52:36,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/03b78c73c05f40d2adff625a7324a8ea as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea 2024-11-15T11:52:36,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T11:52:36,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/07c69275394c41328a7daef88ec0cd7d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d 2024-11-15T11:52:36,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T11:52:36,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/994fa96d3bd54ab3be7652d4660910d0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0 2024-11-15T11:52:36,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T11:52:36,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 962de2b7258c31da99a4c1635a09f927 in 1263ms, sequenceid=286, compaction requested=true 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:36,346 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:36,346 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:36,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:36,347 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:36,347 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:36,347 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:36,347 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,347 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:36,347 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,348 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/96c98667238c4f3fa796c6aabd13a513, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=48.3 K 2024-11-15T11:52:36,348 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0eb3ed95d2b043f3ae415b92cf1b2df2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=48.3 K 2024-11-15T11:52:36,348 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c98667238c4f3fa796c6aabd13a513, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:36,348 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eb3ed95d2b043f3ae415b92cf1b2df2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:36,348 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddde346d69cb47ffba81a22a20a2d602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731671553374 2024-11-15T11:52:36,348 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 71daf825990243a3a6ed035bab8ddc7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731671553374 2024-11-15T11:52:36,349 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cac80767135494aaf506a54d982aff3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731671553738 2024-11-15T11:52:36,349 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e5195ba4c79441795ad2272e1d5701a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731671553738 2024-11-15T11:52:36,350 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03b78c73c05f40d2adff625a7324a8ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:36,350 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c69275394c41328a7daef88ec0cd7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:36,358 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:36,358 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/97503732280e422aa8a9db5ab8e065d9 is 50, key is test_row_0/B:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:36,359 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:36,359 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/2b7c27c013e54a93b883f079c5b96a43 is 50, key is test_row_0/A:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:36,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742134_1310 (size=12983) 2024-11-15T11:52:36,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742135_1311 (size=12983) 2024-11-15T11:52:36,372 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/97503732280e422aa8a9db5ab8e065d9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/97503732280e422aa8a9db5ab8e065d9 2024-11-15T11:52:36,376 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/2b7c27c013e54a93b883f079c5b96a43 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2b7c27c013e54a93b883f079c5b96a43 2024-11-15T11:52:36,381 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 97503732280e422aa8a9db5ab8e065d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:36,381 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=12, startTime=1731671556346; duration=0sec 2024-11-15T11:52:36,381 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 2b7c27c013e54a93b883f079c5b96a43(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:36,381 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=12, startTime=1731671556346; duration=0sec 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:36,381 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:36,383 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:36,383 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:36,383 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,383 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/f040ffb1c38b4554a2be18e44d9b1313, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=48.3 K 2024-11-15T11:52:36,384 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f040ffb1c38b4554a2be18e44d9b1313, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731671552987 2024-11-15T11:52:36,385 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c1915ee5c7ad45a6b71a8f32b1a17632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731671553374 2024-11-15T11:52:36,385 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d38c2e748e64653aab252576caf3811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731671553738 2024-11-15T11:52:36,385 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 994fa96d3bd54ab3be7652d4660910d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:36,397 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#263 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:36,399 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/1ec99f0abc084a1eb3b850d102b0b8cf is 50, key is test_row_0/C:col10/1731671554453/Put/seqid=0 2024-11-15T11:52:36,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742136_1312 (size=12983) 2024-11-15T11:52:36,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:36,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/73479f40f6154736b8a8f00a2aa23c89 is 50, key is test_row_0/A:col10/1731671555096/Put/seqid=0 2024-11-15T11:52:36,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742137_1313 (size=12301) 2024-11-15T11:52:36,449 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/73479f40f6154736b8a8f00a2aa23c89 2024-11-15T11:52:36,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/314218d04cc44f3c98eb994139782807 is 50, key is test_row_0/B:col10/1731671555096/Put/seqid=0 2024-11-15T11:52:36,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742138_1314 (size=12301) 2024-11-15T11:52:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:36,821 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/1ec99f0abc084a1eb3b850d102b0b8cf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1ec99f0abc084a1eb3b850d102b0b8cf 2024-11-15T11:52:36,826 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 1ec99f0abc084a1eb3b850d102b0b8cf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:36,826 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:36,826 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=12, startTime=1731671556346; duration=0sec 2024-11-15T11:52:36,826 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:36,826 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:36,869 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/314218d04cc44f3c98eb994139782807 2024-11-15T11:52:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9e8d60a6404b4915990431ecacf8ede0 is 50, key is test_row_0/C:col10/1731671555096/Put/seqid=0 2024-11-15T11:52:36,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742139_1315 (size=12301) 2024-11-15T11:52:36,897 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9e8d60a6404b4915990431ecacf8ede0 2024-11-15T11:52:36,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/73479f40f6154736b8a8f00a2aa23c89 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89 2024-11-15T11:52:36,907 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T11:52:36,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/314218d04cc44f3c98eb994139782807 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807 2024-11-15T11:52:36,914 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T11:52:36,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9e8d60a6404b4915990431ecacf8ede0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0 2024-11-15T11:52:36,920 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T11:52:36,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 962de2b7258c31da99a4c1635a09f927 in 496ms, sequenceid=301, compaction requested=false 2024-11-15T11:52:36,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:36,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:36,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T11:52:36,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T11:52:36,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T11:52:36,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-15T11:52:36,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5750 sec 2024-11-15T11:52:37,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:37,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:37,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:37,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/6a666c228941470b9a391fb08a5a7689 is 50, key is test_row_0/A:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:37,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742140_1316 (size=14741) 2024-11-15T11:52:37,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671557283, max timestamp=1731671557292Num uniques 859; 2024-11-15T11:52:37,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:37,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671557283, max timestamp=1731671557292Num uniques 859; 2024-11-15T11:52:37,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671557283, max timestamp=1731671557292Num uniques 859; 2024-11-15T11:52:37,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:37,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:37,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671617310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671617311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671617312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671617313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671617316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671617420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671617431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671617431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671617431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671617431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T11:52:37,501 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:37,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T11:52:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:37,503 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:37,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:37,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671617631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671617640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671617640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671617640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671617640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T11:52:37,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:37,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/6a666c228941470b9a391fb08a5a7689 2024-11-15T11:52:37,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/761db077ac9c4c95be29465142c9a914 is 50, key is test_row_0/B:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:37,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742141_1317 (size=12301) 2024-11-15T11:52:37,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/761db077ac9c4c95be29465142c9a914 2024-11-15T11:52:37,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/aa7f2978905246088ef65c633ae43e7d is 50, key is test_row_0/C:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:37,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742142_1318 (size=12301) 2024-11-15T11:52:37,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T11:52:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:37,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671617941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671617950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671617950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671617950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671617950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:37,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T11:52:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:37,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:38,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/aa7f2978905246088ef65c633ae43e7d 2024-11-15T11:52:38,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/6a666c228941470b9a391fb08a5a7689 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689 2024-11-15T11:52:38,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T11:52:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:38,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:38,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689, entries=200, sequenceid=312, filesize=14.4 K 2024-11-15T11:52:38,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/761db077ac9c4c95be29465142c9a914 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914 2024-11-15T11:52:38,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T11:52:38,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/aa7f2978905246088ef65c633ae43e7d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d 2024-11-15T11:52:38,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-15T11:52:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:38,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 962de2b7258c31da99a4c1635a09f927 in 862ms, sequenceid=312, compaction requested=true 2024-11-15T11:52:38,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:38,132 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:38,133 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:38,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:38,133 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:38,133 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:38,133 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,133 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2b7c27c013e54a93b883f079c5b96a43, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=39.1 K 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:38,134 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b7c27c013e54a93b883f079c5b96a43, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:38,134 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/97503732280e422aa8a9db5ab8e065d9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.7 K 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73479f40f6154736b8a8f00a2aa23c89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731671555096 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 97503732280e422aa8a9db5ab8e065d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a666c228941470b9a391fb08a5a7689, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:38,134 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 314218d04cc44f3c98eb994139782807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731671555096 2024-11-15T11:52:38,135 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 761db077ac9c4c95be29465142c9a914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:38,144 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:38,144 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:38,144 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/2358e5b387ec4f80ad38f72549947911 is 50, key is test_row_0/B:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:38,144 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/135ce61020ef4ac596df5d0431106160 is 50, key is test_row_0/A:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:38,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742143_1319 (size=13085) 2024-11-15T11:52:38,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742144_1320 (size=13085) 2024-11-15T11:52:38,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T11:52:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,269 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T11:52:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/3042d58a239a4e0596d4bfa43f2b6578 is 50, key is test_row_0/A:col10/1731671557312/Put/seqid=0 2024-11-15T11:52:38,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742145_1321 (size=12301) 2024-11-15T11:52:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671618467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671618469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671618469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671618476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671618478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,565 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/2358e5b387ec4f80ad38f72549947911 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/2358e5b387ec4f80ad38f72549947911 2024-11-15T11:52:38,565 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/135ce61020ef4ac596df5d0431106160 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/135ce61020ef4ac596df5d0431106160 2024-11-15T11:52:38,570 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 2358e5b387ec4f80ad38f72549947911(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:38,570 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 135ce61020ef4ac596df5d0431106160(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:38,570 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:38,570 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:38,570 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671558132; duration=0sec 2024-11-15T11:52:38,570 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671558133; duration=0sec 2024-11-15T11:52:38,571 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:38,571 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:38,571 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:38,571 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:38,571 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:38,573 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:38,573 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:38,573 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,573 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1ec99f0abc084a1eb3b850d102b0b8cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.7 K 2024-11-15T11:52:38,574 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec99f0abc084a1eb3b850d102b0b8cf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731671554449 2024-11-15T11:52:38,574 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e8d60a6404b4915990431ecacf8ede0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731671555096 2024-11-15T11:52:38,574 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa7f2978905246088ef65c633ae43e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671618580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671618580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,593 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:38,593 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/367a9548c67f4c01ad1f4ef654768fc0 is 50, key is test_row_0/C:col10/1731671557262/Put/seqid=0 2024-11-15T11:52:38,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671618591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671618591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671618591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742146_1322 (size=13085) 2024-11-15T11:52:38,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:38,741 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/3042d58a239a4e0596d4bfa43f2b6578 2024-11-15T11:52:38,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/5cf52f95485445cab77160397dc97fec is 50, key is test_row_0/B:col10/1731671557312/Put/seqid=0 2024-11-15T11:52:38,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742147_1323 (size=12301) 2024-11-15T11:52:38,792 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/5cf52f95485445cab77160397dc97fec 2024-11-15T11:52:38,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671618790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/04bdfcfd41f94d9795d6b0dfd51e15c6 is 50, key is test_row_0/C:col10/1731671557312/Put/seqid=0 2024-11-15T11:52:38,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671618803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671618810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671618811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:38,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671618811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:38,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742148_1324 (size=12301) 2024-11-15T11:52:38,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/04bdfcfd41f94d9795d6b0dfd51e15c6 2024-11-15T11:52:38,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/3042d58a239a4e0596d4bfa43f2b6578 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578 2024-11-15T11:52:38,868 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T11:52:38,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/5cf52f95485445cab77160397dc97fec as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec 2024-11-15T11:52:38,883 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T11:52:38,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/04bdfcfd41f94d9795d6b0dfd51e15c6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6 2024-11-15T11:52:38,903 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6, entries=150, sequenceid=347, filesize=12.0 K 2024-11-15T11:52:38,905 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 962de2b7258c31da99a4c1635a09f927 in 636ms, sequenceid=347, compaction requested=false 2024-11-15T11:52:38,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:38,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:38,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T11:52:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T11:52:38,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T11:52:38,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-15T11:52:38,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4110 sec 2024-11-15T11:52:39,049 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/367a9548c67f4c01ad1f4ef654768fc0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/367a9548c67f4c01ad1f4ef654768fc0 2024-11-15T11:52:39,056 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 367a9548c67f4c01ad1f4ef654768fc0(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:39,056 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:39,056 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671558133; duration=0sec 2024-11-15T11:52:39,056 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:39,056 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:39,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:52:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:39,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:39,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:39,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7634a70cebdb4f81ad77fa997029602f is 50, key is test_row_0/A:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742149_1325 (size=14741) 2024-11-15T11:52:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671619138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671619139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671559132, max timestamp=1731671559141Num uniques 859; 2024-11-15T11:52:39,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:39,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671559132, max timestamp=1731671559141Num uniques 859; 2024-11-15T11:52:39,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671559132, max timestamp=1731671559141Num uniques 859; 2024-11-15T11:52:39,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:39,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:39,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7634a70cebdb4f81ad77fa997029602f 2024-11-15T11:52:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/c4c7872aa32b4218bed3ff50b12e0c19 is 50, key is test_row_0/B:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742150_1326 (size=12301) 2024-11-15T11:52:39,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/c4c7872aa32b4218bed3ff50b12e0c19 2024-11-15T11:52:39,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2a34222cd317451ab91d54b782cf3196 is 50, key is test_row_0/C:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742151_1327 (size=12301) 2024-11-15T11:52:39,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671619198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671619200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671619200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671619250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671619261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671619310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671619310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671619310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671619461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671619470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671619521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671619521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671619522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,572 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:52:39,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2a34222cd317451ab91d54b782cf3196 2024-11-15T11:52:39,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/7634a70cebdb4f81ad77fa997029602f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f 2024-11-15T11:52:39,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f, entries=200, sequenceid=361, filesize=14.4 K 2024-11-15T11:52:39,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/c4c7872aa32b4218bed3ff50b12e0c19 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19 2024-11-15T11:52:39,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19, entries=150, sequenceid=361, filesize=12.0 K 2024-11-15T11:52:39,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2a34222cd317451ab91d54b782cf3196 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196 2024-11-15T11:52:39,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196, entries=150, sequenceid=361, filesize=12.0 K 2024-11-15T11:52:39,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 962de2b7258c31da99a4c1635a09f927 in 502ms, sequenceid=361, compaction requested=true 2024-11-15T11:52:39,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:39,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:39,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:39,617 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:39,618 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:39,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:39,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:39,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:39,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:39,619 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:39,619 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:39,620 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,620 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/135ce61020ef4ac596df5d0431106160, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=39.2 K 2024-11-15T11:52:39,620 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:39,620 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:39,620 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,620 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/2358e5b387ec4f80ad38f72549947911, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.8 K 2024-11-15T11:52:39,620 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135ce61020ef4ac596df5d0431106160, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:39,621 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2358e5b387ec4f80ad38f72549947911, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:39,621 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3042d58a239a4e0596d4bfa43f2b6578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731671557307 2024-11-15T11:52:39,621 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf52f95485445cab77160397dc97fec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731671557307 2024-11-15T11:52:39,621 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7634a70cebdb4f81ad77fa997029602f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:39,622 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c7872aa32b4218bed3ff50b12e0c19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:39,632 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:39,632 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/b9a0014628ee47cb924debe427b37f34 is 50, key is test_row_0/B:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,640 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:39,640 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/2672d21d04784e95a616d54f262218b5 is 50, key is test_row_0/A:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T11:52:39,650 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:39,651 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:39,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-15T11:52:39,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:39,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:39,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:39,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:39,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742152_1328 (size=13187) 2024-11-15T11:52:39,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742153_1329 (size=13187) 2024-11-15T11:52:39,674 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/b9a0014628ee47cb924debe427b37f34 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/b9a0014628ee47cb924debe427b37f34 2024-11-15T11:52:39,679 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into b9a0014628ee47cb924debe427b37f34(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:39,679 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:39,679 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671559617; duration=0sec 2024-11-15T11:52:39,679 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:39,679 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:39,679 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:39,680 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:39,680 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:39,680 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,680 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/367a9548c67f4c01ad1f4ef654768fc0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=36.8 K 2024-11-15T11:52:39,680 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 367a9548c67f4c01ad1f4ef654768fc0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731671557262 2024-11-15T11:52:39,680 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 04bdfcfd41f94d9795d6b0dfd51e15c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731671557307 2024-11-15T11:52:39,681 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a34222cd317451ab91d54b782cf3196, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:39,691 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:39,691 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2de5a1be47334ee7b29a0a96e6f0c942 is 50, key is test_row_0/C:col10/1731671558473/Put/seqid=0 2024-11-15T11:52:39,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742154_1330 (size=13187) 2024-11-15T11:52:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:39,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:39,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:39,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0e98406d593e4831b806679ea23308af is 50, key is test_row_0/A:col10/1731671559198/Put/seqid=0 2024-11-15T11:52:39,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742155_1331 (size=12301) 2024-11-15T11:52:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671619793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671619797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671619830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671619830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671619830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671619900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671619910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:39,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:39,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:39,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:40,075 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/2672d21d04784e95a616d54f262218b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2672d21d04784e95a616d54f262218b5 2024-11-15T11:52:40,079 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 2672d21d04784e95a616d54f262218b5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:40,079 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:40,080 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671559617; duration=0sec 2024-11-15T11:52:40,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:40,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:40,108 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/2de5a1be47334ee7b29a0a96e6f0c942 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2de5a1be47334ee7b29a0a96e6f0c942 2024-11-15T11:52:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671620111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,116 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 2de5a1be47334ee7b29a0a96e6f0c942(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:40,116 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:40,116 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671559618; duration=0sec 2024-11-15T11:52:40,116 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:40,116 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:40,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671620120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0e98406d593e4831b806679ea23308af 2024-11-15T11:52:40,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/888e5bb4ba3746be8665ffae2a11389d is 50, key is test_row_0/B:col10/1731671559198/Put/seqid=0 2024-11-15T11:52:40,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742156_1332 (size=12301) 2024-11-15T11:52:40,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671620341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671620341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671620341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671620422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671620430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/888e5bb4ba3746be8665ffae2a11389d 2024-11-15T11:52:40,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/db37a11b8b894072af30cf9a525a8641 is 50, key is test_row_0/C:col10/1731671559198/Put/seqid=0 2024-11-15T11:52:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742157_1333 (size=12301) 2024-11-15T11:52:40,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:40,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:40,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671620930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:40,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671620940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:40,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/db37a11b8b894072af30cf9a525a8641 2024-11-15T11:52:41,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0e98406d593e4831b806679ea23308af as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af 2024-11-15T11:52:41,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T11:52:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/888e5bb4ba3746be8665ffae2a11389d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d 2024-11-15T11:52:41,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T11:52:41,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/db37a11b8b894072af30cf9a525a8641 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641 2024-11-15T11:52:41,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:41,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:41,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641, entries=150, sequenceid=400, filesize=12.0 K 2024-11-15T11:52:41,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 962de2b7258c31da99a4c1635a09f927 in 1260ms, sequenceid=400, compaction requested=false 2024-11-15T11:52:41,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:41,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,184 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0afcff2dac9a4d1589a5d9da4f1413b5 is 50, key is test_row_0/A:col10/1731671559796/Put/seqid=0 2024-11-15T11:52:41,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742158_1334 (size=9857) 2024-11-15T11:52:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:41,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671561368, max timestamp=1731671561378Num uniques 859; 2024-11-15T11:52:41,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:41,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671561368, max timestamp=1731671561378Num uniques 859; 2024-11-15T11:52:41,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671561368, max timestamp=1731671561378Num uniques 859; 2024-11-15T11:52:41,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:41,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671561409, max timestamp=1731671561433Num uniques 1068; 2024-11-15T11:52:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T11:52:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671621429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671561409, max timestamp=1731671561433Num uniques 1068; 2024-11-15T11:52:41,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671561409, max timestamp=1731671561433Num uniques 1068; 2024-11-15T11:52:41,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T11:52:41,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-15T11:52:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671621492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671621495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671621540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,593 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0afcff2dac9a4d1589a5d9da4f1413b5 2024-11-15T11:52:41,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671621601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671621601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/35d7ca9f214744578f036fbcf4ce3fb3 is 50, key is test_row_0/B:col10/1731671559796/Put/seqid=0 2024-11-15T11:52:41,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742159_1335 (size=9857) 2024-11-15T11:52:41,620 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/35d7ca9f214744578f036fbcf4ce3fb3 2024-11-15T11:52:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/400731be96dd44cd9c050673e919abc7 is 50, key is test_row_0/C:col10/1731671559796/Put/seqid=0 2024-11-15T11:52:41,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742160_1336 (size=9857) 2024-11-15T11:52:41,636 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/400731be96dd44cd9c050673e919abc7 2024-11-15T11:52:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/0afcff2dac9a4d1589a5d9da4f1413b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5 2024-11-15T11:52:41,644 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T11:52:41,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/35d7ca9f214744578f036fbcf4ce3fb3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3 2024-11-15T11:52:41,650 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T11:52:41,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/400731be96dd44cd9c050673e919abc7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7 2024-11-15T11:52:41,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7, entries=100, sequenceid=411, filesize=9.6 K 2024-11-15T11:52:41,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 962de2b7258c31da99a4c1635a09f927 in 473ms, sequenceid=411, compaction requested=true 2024-11-15T11:52:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-15T11:52:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-15T11:52:41,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T11:52:41,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0050 sec 2024-11-15T11:52:41,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0110 sec 2024-11-15T11:52:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:41,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:41,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/80e5639acbc74268935d9f4c799f005d is 50, key is test_row_0/A:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:41,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742161_1337 (size=14741) 2024-11-15T11:52:41,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/80e5639acbc74268935d9f4c799f005d 2024-11-15T11:52:41,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/258bd05fd95242b6958080a947c2d76f is 50, key is test_row_0/B:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:41,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671621781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742162_1338 (size=12301) 2024-11-15T11:52:41,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/258bd05fd95242b6958080a947c2d76f 2024-11-15T11:52:41,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/35b42d454c714053bad319cff64031ee is 50, key is test_row_0/C:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:41,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742163_1339 (size=12301) 2024-11-15T11:52:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T11:52:41,810 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:41,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-15T11:52:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:41,812 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:41,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671621810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,813 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671621810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671621891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:41,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671621941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,947 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671621950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:41,953 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 962de2b7258c31da99a4c1635a09f927 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:41,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:41,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T11:52:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:41,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671622100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T11:52:42,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:42,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:42,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671622120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671622120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:42,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/35b42d454c714053bad319cff64031ee 2024-11-15T11:52:42,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/80e5639acbc74268935d9f4c799f005d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d 2024-11-15T11:52:42,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d, entries=200, sequenceid=466, filesize=14.4 K 2024-11-15T11:52:42,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/258bd05fd95242b6958080a947c2d76f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f 2024-11-15T11:52:42,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f, entries=150, sequenceid=466, filesize=12.0 K 2024-11-15T11:52:42,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/35b42d454c714053bad319cff64031ee as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee 2024-11-15T11:52:42,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee, entries=150, sequenceid=466, filesize=12.0 K 2024-11-15T11:52:42,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 962de2b7258c31da99a4c1635a09f927 in 463ms, sequenceid=466, compaction requested=true 2024-11-15T11:52:42,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:42,218 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:42,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:42,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:42,219 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:42,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:42,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:42,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:42,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:42,221 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50086 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:42,221 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:42,221 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,221 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2672d21d04784e95a616d54f262218b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=48.9 K 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2672d21d04784e95a616d54f262218b5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:42,222 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,222 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/b9a0014628ee47cb924debe427b37f34, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=46.5 K 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b9a0014628ee47cb924debe427b37f34, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e98406d593e4831b806679ea23308af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731671559157 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 888e5bb4ba3746be8665ffae2a11389d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731671559157 2024-11-15T11:52:42,222 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0afcff2dac9a4d1589a5d9da4f1413b5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731671559785 2024-11-15T11:52:42,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d7ca9f214744578f036fbcf4ce3fb3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731671559785 2024-11-15T11:52:42,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e5639acbc74268935d9f4c799f005d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561487 2024-11-15T11:52:42,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 258bd05fd95242b6958080a947c2d76f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561491 2024-11-15T11:52:42,237 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:42,238 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/44b6b370fee0438fbda90950bf60c1ee is 50, key is test_row_0/A:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:42,241 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:42,241 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/d1f4049373794d84ba9a15d4f4247663 is 50, key is test_row_0/B:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742164_1340 (size=13323) 2024-11-15T11:52:42,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742165_1341 (size=13323) 2024-11-15T11:52:42,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,271 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:42,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/47ce2ecaf3ac4e89981a6b5126ae9003 is 50, key is test_row_0/A:col10/1731671561764/Put/seqid=0 2024-11-15T11:52:42,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742166_1342 (size=12301) 2024-11-15T11:52:42,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671562455, max timestamp=1731671562497Num uniques 859; 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671562455, max timestamp=1731671562497Num uniques 859; 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671562455, max timestamp=1731671562497Num uniques 859; 2024-11-15T11:52:42,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671562566, max timestamp=1731671562601Num uniques 1018; 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671562566, max timestamp=1731671562601Num uniques 1018; 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671562566, max timestamp=1731671562601Num uniques 1018; 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T11:52:42,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-15T11:52:42,665 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/d1f4049373794d84ba9a15d4f4247663 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/d1f4049373794d84ba9a15d4f4247663 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731671562636, max timestamp=1731671562661Num uniques 1027; 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731671562636, max timestamp=1731671562661Num uniques 1027; 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731671562636, max timestamp=1731671562661Num uniques 1027; 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T11:52:42,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-15T11:52:42,670 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/44b6b370fee0438fbda90950bf60c1ee as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/44b6b370fee0438fbda90950bf60c1ee 2024-11-15T11:52:42,672 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into d1f4049373794d84ba9a15d4f4247663(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:42,672 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:42,673 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=12, startTime=1731671562219; duration=0sec 2024-11-15T11:52:42,673 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:42,673 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:42,673 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:52:42,675 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:52:42,675 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:42,675 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:42,675 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2de5a1be47334ee7b29a0a96e6f0c942, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=46.5 K 2024-11-15T11:52:42,675 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de5a1be47334ee7b29a0a96e6f0c942, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731671558467 2024-11-15T11:52:42,676 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting db37a11b8b894072af30cf9a525a8641, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731671559157 2024-11-15T11:52:42,676 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 400731be96dd44cd9c050673e919abc7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731671559785 2024-11-15T11:52:42,676 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b42d454c714053bad319cff64031ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561491 2024-11-15T11:52:42,679 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into 44b6b370fee0438fbda90950bf60c1ee(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:42,679 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:42,679 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=12, startTime=1731671562218; duration=0sec 2024-11-15T11:52:42,679 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:42,679 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:42,685 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:42,686 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/8d8338a0dd6d4caaa2096daf73db2b97 is 50, key is test_row_0/C:col10/1731671561491/Put/seqid=0 2024-11-15T11:52:42,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/47ce2ecaf3ac4e89981a6b5126ae9003 2024-11-15T11:52:42,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742167_1343 (size=13323) 2024-11-15T11:52:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f1922a2ca92d45aa9cddace1a3253f0b is 50, key is test_row_0/B:col10/1731671561764/Put/seqid=0 2024-11-15T11:52:42,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671622706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742168_1344 (size=12301) 2024-11-15T11:52:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671622708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671622709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671622820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671622820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671622820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:42,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671623030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671623030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671623030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,108 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/8d8338a0dd6d4caaa2096daf73db2b97 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8d8338a0dd6d4caaa2096daf73db2b97 2024-11-15T11:52:43,113 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 8d8338a0dd6d4caaa2096daf73db2b97(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:43,113 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:43,113 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=12, startTime=1731671562219; duration=0sec 2024-11-15T11:52:43,113 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:43,113 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:43,115 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f1922a2ca92d45aa9cddace1a3253f0b 2024-11-15T11:52:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/d1ff65b67e5041139bab28cda9cbbe1a is 50, key is test_row_0/C:col10/1731671561764/Put/seqid=0 2024-11-15T11:52:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742169_1345 (size=12301) 2024-11-15T11:52:43,133 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/d1ff65b67e5041139bab28cda9cbbe1a 2024-11-15T11:52:43,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/47ce2ecaf3ac4e89981a6b5126ae9003 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003 2024-11-15T11:52:43,141 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003, entries=150, sequenceid=474, filesize=12.0 K 2024-11-15T11:52:43,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f1922a2ca92d45aa9cddace1a3253f0b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b 2024-11-15T11:52:43,145 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b, entries=150, sequenceid=474, filesize=12.0 K 2024-11-15T11:52:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/d1ff65b67e5041139bab28cda9cbbe1a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a 2024-11-15T11:52:43,150 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a, entries=150, sequenceid=474, filesize=12.0 K 2024-11-15T11:52:43,151 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 962de2b7258c31da99a4c1635a09f927 in 880ms, sequenceid=474, compaction requested=false 2024-11-15T11:52:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:43,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-15T11:52:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-15T11:52:43,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T11:52:43,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3390 sec 2024-11-15T11:52:43,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3430 sec 2024-11-15T11:52:43,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:43,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=154.31 KB heapSize=434.39 KB 2024-11-15T11:52:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:43,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:52:43,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bdb5063a5efc4122b7feb582e9adaf89 is 50, key is test_row_0/A:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742170_1346 (size=14741) 2024-11-15T11:52:43,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671623351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671623353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671623353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671623461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671623461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671623461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671623670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671623670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671623670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:43,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bdb5063a5efc4122b7feb582e9adaf89 2024-11-15T11:52:43,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/8fc2bd0e5d294567bdabbf985c4f0821 is 50, key is test_row_0/B:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742171_1347 (size=12301) 2024-11-15T11:52:43,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/8fc2bd0e5d294567bdabbf985c4f0821 2024-11-15T11:52:43,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c77ef0bf85a24109a55355e6c3645140 is 50, key is test_row_0/C:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742172_1348 (size=12301) 2024-11-15T11:52:43,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=549 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c77ef0bf85a24109a55355e6c3645140 2024-11-15T11:52:43,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bdb5063a5efc4122b7feb582e9adaf89 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89 2024-11-15T11:52:43,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89, entries=200, sequenceid=549, filesize=14.4 K 2024-11-15T11:52:43,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/8fc2bd0e5d294567bdabbf985c4f0821 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821 2024-11-15T11:52:43,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821, entries=150, sequenceid=549, filesize=12.0 K 2024-11-15T11:52:43,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/c77ef0bf85a24109a55355e6c3645140 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140 2024-11-15T11:52:43,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140, entries=150, sequenceid=549, filesize=12.0 K 2024-11-15T11:52:43,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~434.34 KB/444768, currentSize=33.54 KB/34350 for 962de2b7258c31da99a4c1635a09f927 in 472ms, sequenceid=549, compaction requested=true 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:43,817 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:43,817 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:43,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:43,818 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:43,818 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/A is initiating minor compaction (all files) 2024-11-15T11:52:43,818 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/A in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:43,818 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/44b6b370fee0438fbda90950bf60c1ee, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=39.4 K 2024-11-15T11:52:43,818 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:43,818 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/B is initiating minor compaction (all files) 2024-11-15T11:52:43,818 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/B in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:43,818 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b6b370fee0438fbda90950bf60c1ee, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561491 2024-11-15T11:52:43,818 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/d1f4049373794d84ba9a15d4f4247663, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=37.0 K 2024-11-15T11:52:43,819 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47ce2ecaf3ac4e89981a6b5126ae9003, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731671561764 2024-11-15T11:52:43,819 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f4049373794d84ba9a15d4f4247663, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561491 2024-11-15T11:52:43,820 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdb5063a5efc4122b7feb582e9adaf89, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731671562701 2024-11-15T11:52:43,820 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f1922a2ca92d45aa9cddace1a3253f0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731671561764 2024-11-15T11:52:43,820 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc2bd0e5d294567bdabbf985c4f0821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731671562701 2024-11-15T11:52:43,830 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#A#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:43,830 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/df8e35fe82a24687b788dc74c8ed0b08 is 50, key is test_row_0/A:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,831 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#B#compaction#301 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:43,831 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/302915cd2dd6411dbe20b9792a99dd95 is 50, key is test_row_0/B:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742173_1349 (size=13425) 2024-11-15T11:52:43,847 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/df8e35fe82a24687b788dc74c8ed0b08 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/df8e35fe82a24687b788dc74c8ed0b08 2024-11-15T11:52:43,854 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/A of 962de2b7258c31da99a4c1635a09f927 into df8e35fe82a24687b788dc74c8ed0b08(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:43,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:43,854 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/A, priority=13, startTime=1731671563817; duration=0sec 2024-11-15T11:52:43,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:43,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:43,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:43,856 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:43,856 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 962de2b7258c31da99a4c1635a09f927/C is initiating minor compaction (all files) 2024-11-15T11:52:43,856 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 962de2b7258c31da99a4c1635a09f927/C in TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:43,856 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8d8338a0dd6d4caaa2096daf73db2b97, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp, totalSize=37.0 K 2024-11-15T11:52:43,856 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8338a0dd6d4caaa2096daf73db2b97, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1731671561491 2024-11-15T11:52:43,857 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ff65b67e5041139bab28cda9cbbe1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731671561764 2024-11-15T11:52:43,857 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77ef0bf85a24109a55355e6c3645140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=549, earliestPutTs=1731671562701 2024-11-15T11:52:43,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742174_1350 (size=13425) 2024-11-15T11:52:43,878 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/302915cd2dd6411dbe20b9792a99dd95 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/302915cd2dd6411dbe20b9792a99dd95 2024-11-15T11:52:43,888 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 962de2b7258c31da99a4c1635a09f927#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:43,889 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/4b0f93e35044454c97cdefe4ee8378da is 50, key is test_row_0/C:col10/1731671562701/Put/seqid=0 2024-11-15T11:52:43,890 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/B of 962de2b7258c31da99a4c1635a09f927 into 302915cd2dd6411dbe20b9792a99dd95(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:43,890 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:43,890 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/B, priority=13, startTime=1731671563817; duration=0sec 2024-11-15T11:52:43,890 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:43,890 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:43,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742175_1351 (size=13425) 2024-11-15T11:52:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-15T11:52:43,961 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:43,961 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-15T11:52:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:43,963 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:43,964 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:43,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:43,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:43,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:43,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/baedfaf94fea4dd18c3f22523643ad3b is 50, key is test_row_0/A:col10/1731671563974/Put/seqid=0 2024-11-15T11:52:43,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742176_1352 (size=12301) 2024-11-15T11:52:44,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/baedfaf94fea4dd18c3f22523643ad3b 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671563992, max timestamp=1731671564002Num uniques 859; 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671563992, max timestamp=1731671564002Num uniques 859; 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671563992, max timestamp=1731671564002Num uniques 859; 2024-11-15T11:52:44,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:52:44,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/934db9e184f6421bb91be5d180d7bda4 is 50, key is test_row_0/B:col10/1731671563974/Put/seqid=0 2024-11-15T11:52:44,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742177_1353 (size=12301) 2024-11-15T11:52:44,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/934db9e184f6421bb91be5d180d7bda4 2024-11-15T11:52:44,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/871ee35c2c704a30b35e7987875b2457 is 50, key is test_row_0/C:col10/1731671563974/Put/seqid=0 2024-11-15T11:52:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742178_1354 (size=12301) 2024-11-15T11:52:44,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671624021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671624022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671564017, max timestamp=1731671564032Num uniques 1018; 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671564017, max timestamp=1731671564032Num uniques 1018; 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731671564017, max timestamp=1731671564032Num uniques 1018; 2024-11-15T11:52:44,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-15T11:52:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:44,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671624087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671624091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671624092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T11:52:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671624161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671624161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671624200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671624200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671624200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T11:52:44,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:44,297 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/4b0f93e35044454c97cdefe4ee8378da as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/4b0f93e35044454c97cdefe4ee8378da 2024-11-15T11:52:44,302 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 962de2b7258c31da99a4c1635a09f927/C of 962de2b7258c31da99a4c1635a09f927 into 4b0f93e35044454c97cdefe4ee8378da(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:44,302 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:44,302 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., storeName=962de2b7258c31da99a4c1635a09f927/C, priority=13, startTime=1731671563817; duration=0sec 2024-11-15T11:52:44,303 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:44,303 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671624370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671624370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671624411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671624411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671624411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,416 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T11:52:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:44,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/871ee35c2c704a30b35e7987875b2457 2024-11-15T11:52:44,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/baedfaf94fea4dd18c3f22523643ad3b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/baedfaf94fea4dd18c3f22523643ad3b 2024-11-15T11:52:44,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/baedfaf94fea4dd18c3f22523643ad3b, entries=150, sequenceid=561, filesize=12.0 K 2024-11-15T11:52:44,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/934db9e184f6421bb91be5d180d7bda4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/934db9e184f6421bb91be5d180d7bda4 2024-11-15T11:52:44,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/934db9e184f6421bb91be5d180d7bda4, entries=150, sequenceid=561, filesize=12.0 K 2024-11-15T11:52:44,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/871ee35c2c704a30b35e7987875b2457 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/871ee35c2c704a30b35e7987875b2457 2024-11-15T11:52:44,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/871ee35c2c704a30b35e7987875b2457, entries=150, sequenceid=561, filesize=12.0 K 2024-11-15T11:52:44,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 962de2b7258c31da99a4c1635a09f927 in 479ms, sequenceid=561, compaction requested=false 2024-11-15T11:52:44,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:44,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-15T11:52:44,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:44,581 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-15T11:52:44,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:44,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:44,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:44,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/26a49000136746c68f1808acb4c772b5 is 50, key is test_row_0/A:col10/1731671564070/Put/seqid=0 2024-11-15T11:52:44,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742179_1355 (size=12301) 2024-11-15T11:52:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:44,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/26a49000136746c68f1808acb4c772b5 2024-11-15T11:52:44,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/4a9147688d8f4f0f98d551d554bc02ed is 50, key is test_row_0/B:col10/1731671564070/Put/seqid=0 2024-11-15T11:52:44,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742180_1356 (size=12301) 2024-11-15T11:52:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:44,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. as already flushing 2024-11-15T11:52:44,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671624696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671624701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671624721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671624721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671624721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671624811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:44,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:44,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671624811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/4a9147688d8f4f0f98d551d554bc02ed 2024-11-15T11:52:45,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/342fac1048374f4facf2f3bf29652d95 is 50, key is test_row_0/C:col10/1731671564070/Put/seqid=0 2024-11-15T11:52:45,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742181_1357 (size=12301) 2024-11-15T11:52:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671625020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671625020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:45,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39088 deadline: 1731671625230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39056 deadline: 1731671625231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39110 deadline: 1731671625231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39094 deadline: 1731671625331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39076 deadline: 1731671625331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:52:45,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/342fac1048374f4facf2f3bf29652d95 2024-11-15T11:52:45,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/26a49000136746c68f1808acb4c772b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/26a49000136746c68f1808acb4c772b5 2024-11-15T11:52:45,424 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/26a49000136746c68f1808acb4c772b5, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T11:52:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/4a9147688d8f4f0f98d551d554bc02ed as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4a9147688d8f4f0f98d551d554bc02ed 2024-11-15T11:52:45,430 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4a9147688d8f4f0f98d551d554bc02ed, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T11:52:45,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/342fac1048374f4facf2f3bf29652d95 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/342fac1048374f4facf2f3bf29652d95 2024-11-15T11:52:45,437 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/342fac1048374f4facf2f3bf29652d95, entries=150, sequenceid=614, filesize=12.0 K 2024-11-15T11:52:45,438 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=46.96 KB/48090 for 962de2b7258c31da99a4c1635a09f927 in 857ms, sequenceid=614, compaction requested=true 2024-11-15T11:52:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-15T11:52:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-15T11:52:45,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-15T11:52:45,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-15T11:52:45,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.4890 sec 2024-11-15T11:52:45,608 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-15T11:52:45,608 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,608 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,609 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,610 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-15T11:52:45,610 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,610 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,610 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,610 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,611 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-15T11:52:45,611 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,611 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,611 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,611 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-11-15T11:52:45,611 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,611 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,611 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,612 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-15T11:52:45,612 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,612 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,612 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:45,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:45,847 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:45,847 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-15T11:52:45,847 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:45,847 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,847 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:45,847 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:45,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:45,847 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,848 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:45,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:45,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/de937f8d9fad487f8cb64358a8b563d1 is 50, key is test_row_0/A:col10/1731671565843/Put/seqid=0 2024-11-15T11:52:45,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742182_1358 (size=12301) 2024-11-15T11:52:46,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T11:52:46,111 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:46,242 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-15T11:52:46,242 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:46,242 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,242 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:46,253 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-15T11:52:46,253 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:46,253 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-15T11:52:46,253 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,253 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:52:46,253 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,253 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:46,253 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 185 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2797 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2800 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2707 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2809 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2810 2024-11-15T11:52:46,254 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:52:46,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:52:46,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:52:46,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:46,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:52:46,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:52:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:46,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/de937f8d9fad487f8cb64358a8b563d1 2024-11-15T11:52:46,259 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671566259"}]},"ts":"1731671566259"} 2024-11-15T11:52:46,260 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:52:46,260 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:52:46,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:52:46,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, UNASSIGN}] 2024-11-15T11:52:46,264 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, UNASSIGN 2024-11-15T11:52:46,265 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=962de2b7258c31da99a4c1635a09f927, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:46,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, UNASSIGN because future has completed 2024-11-15T11:52:46,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/458a718d5ae74fbe9394379a5c722b06 is 50, key is test_row_0/B:col10/1731671565843/Put/seqid=0 2024-11-15T11:52:46,266 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:52:46,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:46,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742183_1359 (size=12301) 2024-11-15T11:52:46,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/458a718d5ae74fbe9394379a5c722b06 2024-11-15T11:52:46,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9eb79c269eed4a2492123f85ac1ee850 is 50, key is test_row_0/C:col10/1731671565843/Put/seqid=0 2024-11-15T11:52:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742184_1360 (size=12301) 2024-11-15T11:52:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:46,420 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:46,420 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:52:46,420 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 962de2b7258c31da99a4c1635a09f927, disabling compactions & flushes 2024-11-15T11:52:46,420 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:46,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9eb79c269eed4a2492123f85ac1ee850 2024-11-15T11:52:46,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/de937f8d9fad487f8cb64358a8b563d1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/de937f8d9fad487f8cb64358a8b563d1 2024-11-15T11:52:46,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/de937f8d9fad487f8cb64358a8b563d1, entries=150, sequenceid=626, filesize=12.0 K 2024-11-15T11:52:46,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/458a718d5ae74fbe9394379a5c722b06 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/458a718d5ae74fbe9394379a5c722b06 2024-11-15T11:52:46,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/458a718d5ae74fbe9394379a5c722b06, entries=150, sequenceid=626, filesize=12.0 K 2024-11-15T11:52:46,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/9eb79c269eed4a2492123f85ac1ee850 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9eb79c269eed4a2492123f85ac1ee850 2024-11-15T11:52:46,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9eb79c269eed4a2492123f85ac1ee850, entries=150, sequenceid=626, filesize=12.0 K 2024-11-15T11:52:46,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 962de2b7258c31da99a4c1635a09f927 in 851ms, sequenceid=626, compaction requested=true 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 962de2b7258c31da99a4c1635a09f927: 2024-11-15T11:52:46,697 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. because compaction request was cancelled 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. after waiting 0 ms 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:A 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:B, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. because compaction request was cancelled 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:B 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:46,697 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 962de2b7258c31da99a4c1635a09f927 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 962de2b7258c31da99a4c1635a09f927:C, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. because compaction request was cancelled 2024-11-15T11:52:46,697 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 962de2b7258c31da99a4c1635a09f927:C 2024-11-15T11:52:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=A 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=B 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 962de2b7258c31da99a4c1635a09f927, store=C 2024-11-15T11:52:46,697 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:46,700 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bd907c72258c44519201179059d5b022 is 50, key is test_row_0/A:col10/1731671566251/Put/seqid=0 2024-11-15T11:52:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742185_1361 (size=9857) 2024-11-15T11:52:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:47,103 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bd907c72258c44519201179059d5b022 2024-11-15T11:52:47,110 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f0806c4ced854ef3b11acf2468bb7d9e is 50, key is test_row_0/B:col10/1731671566251/Put/seqid=0 2024-11-15T11:52:47,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742186_1362 (size=9857) 2024-11-15T11:52:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:47,517 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f0806c4ced854ef3b11acf2468bb7d9e 2024-11-15T11:52:47,524 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5a3a6679ed024885ae56b328c373f15f is 50, key is test_row_0/C:col10/1731671566251/Put/seqid=0 2024-11-15T11:52:47,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742187_1363 (size=9857) 2024-11-15T11:52:47,928 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=632 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5a3a6679ed024885ae56b328c373f15f 2024-11-15T11:52:47,938 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/A/bd907c72258c44519201179059d5b022 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bd907c72258c44519201179059d5b022 2024-11-15T11:52:47,942 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bd907c72258c44519201179059d5b022, entries=100, sequenceid=632, filesize=9.6 K 2024-11-15T11:52:47,944 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/B/f0806c4ced854ef3b11acf2468bb7d9e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f0806c4ced854ef3b11acf2468bb7d9e 2024-11-15T11:52:47,948 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f0806c4ced854ef3b11acf2468bb7d9e, entries=100, sequenceid=632, filesize=9.6 K 2024-11-15T11:52:47,949 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/.tmp/C/5a3a6679ed024885ae56b328c373f15f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5a3a6679ed024885ae56b328c373f15f 2024-11-15T11:52:47,953 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5a3a6679ed024885ae56b328c373f15f, entries=100, sequenceid=632, filesize=9.6 K 2024-11-15T11:52:47,954 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 962de2b7258c31da99a4c1635a09f927 in 1257ms, sequenceid=632, compaction requested=true 2024-11-15T11:52:47,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/f6f5ae9701224ed5970181c3b954aa41, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a8b33a769184e54b6709431ca44e40a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/260d952ca1e24834968dbf2ca773744a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/c7070ceac1c34c7b8182cae7c120a6f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0eb3ed95d2b043f3ae415b92cf1b2df2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2b7c27c013e54a93b883f079c5b96a43, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/135ce61020ef4ac596df5d0431106160, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2672d21d04784e95a616d54f262218b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/44b6b370fee0438fbda90950bf60c1ee, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89] to archive 2024-11-15T11:52:47,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:47,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/d22c355bb7f046c595391d9ce464fb2c 2024-11-15T11:52:47,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7c7d4eb6d2e346788599b08b17f1f69b 2024-11-15T11:52:47,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8d97097a1ede4a91b1644f87ef247832 2024-11-15T11:52:47,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/f6f5ae9701224ed5970181c3b954aa41 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/f6f5ae9701224ed5970181c3b954aa41 2024-11-15T11:52:47,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e3fc3ef2c8044659abc565efbdc948d 2024-11-15T11:52:47,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a8b33a769184e54b6709431ca44e40a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a8b33a769184e54b6709431ca44e40a 2024-11-15T11:52:47,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/1e699dc8bde5420eb1f39c3fbc95e0fe 2024-11-15T11:52:47,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/60ff7713d79944bdb26e0c3539ee7be6 2024-11-15T11:52:47,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/9ac323ba0be8478fbacc23f1a532daeb 2024-11-15T11:52:47,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/8a45bd099714405fad107d23ef211ff6 2024-11-15T11:52:47,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/260d952ca1e24834968dbf2ca773744a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/260d952ca1e24834968dbf2ca773744a 2024-11-15T11:52:47,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/e66eb60c72aa40d58374f6fc934c2182 2024-11-15T11:52:47,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/c7070ceac1c34c7b8182cae7c120a6f1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/c7070ceac1c34c7b8182cae7c120a6f1 2024-11-15T11:52:47,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0d73bb0324cc412cba37aff04d3a816b 2024-11-15T11:52:47,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/b126d9c2b99d469286c461eb54afac65 2024-11-15T11:52:47,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0eb3ed95d2b043f3ae415b92cf1b2df2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0eb3ed95d2b043f3ae415b92cf1b2df2 2024-11-15T11:52:47,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/39165bc4e01d43609f3e585e4857a703 2024-11-15T11:52:47,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/ddde346d69cb47ffba81a22a20a2d602 2024-11-15T11:52:47,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/4cac80767135494aaf506a54d982aff3 2024-11-15T11:52:47,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2b7c27c013e54a93b883f079c5b96a43 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2b7c27c013e54a93b883f079c5b96a43 2024-11-15T11:52:47,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/03b78c73c05f40d2adff625a7324a8ea 2024-11-15T11:52:47,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/73479f40f6154736b8a8f00a2aa23c89 2024-11-15T11:52:47,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/6a666c228941470b9a391fb08a5a7689 2024-11-15T11:52:47,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/135ce61020ef4ac596df5d0431106160 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/135ce61020ef4ac596df5d0431106160 2024-11-15T11:52:47,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/3042d58a239a4e0596d4bfa43f2b6578 2024-11-15T11:52:47,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/7634a70cebdb4f81ad77fa997029602f 2024-11-15T11:52:47,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2672d21d04784e95a616d54f262218b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/2672d21d04784e95a616d54f262218b5 2024-11-15T11:52:47,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0e98406d593e4831b806679ea23308af 2024-11-15T11:52:47,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/0afcff2dac9a4d1589a5d9da4f1413b5 2024-11-15T11:52:47,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/80e5639acbc74268935d9f4c799f005d 2024-11-15T11:52:47,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/44b6b370fee0438fbda90950bf60c1ee to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/44b6b370fee0438fbda90950bf60c1ee 2024-11-15T11:52:47,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/47ce2ecaf3ac4e89981a6b5126ae9003 2024-11-15T11:52:47,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bdb5063a5efc4122b7feb582e9adaf89 2024-11-15T11:52:48,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8e373623d01042ca802a79ade0d38c6c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e52861b7f780435eb839a58c84456916, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1f0a5ed122dc4d7caddbb1d683875a40, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4ee27aceee964651a3a75f927751a08d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/96c98667238c4f3fa796c6aabd13a513, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/97503732280e422aa8a9db5ab8e065d9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/2358e5b387ec4f80ad38f72549947911, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/b9a0014628ee47cb924debe427b37f34, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/d1f4049373794d84ba9a15d4f4247663, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821] to archive 2024-11-15T11:52:48,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:48,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e6c2bdfc886427a8cf12da4d25e9913 2024-11-15T11:52:48,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/a988054f6bed4b75ae9b045212bd83b3 2024-11-15T11:52:48,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8e373623d01042ca802a79ade0d38c6c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8e373623d01042ca802a79ade0d38c6c 2024-11-15T11:52:48,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/45dc944f30c04b51a88565cd31fbf06e 2024-11-15T11:52:48,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/38c49da9555d4b9f8e17c0d1c6f6e250 2024-11-15T11:52:48,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e52861b7f780435eb839a58c84456916 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e52861b7f780435eb839a58c84456916 2024-11-15T11:52:48,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/467b91b339094f9c8de8b0c6c7f5009e 2024-11-15T11:52:48,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/58d4ed6995724c27a7983b1f5421b25e 2024-11-15T11:52:48,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1573143ba95a46449e24c6f2723990cf 2024-11-15T11:52:48,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1f0a5ed122dc4d7caddbb1d683875a40 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/1f0a5ed122dc4d7caddbb1d683875a40 2024-11-15T11:52:48,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/23e47c57e8a04f029b5ee34f622e10c2 2024-11-15T11:52:48,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/21428fed0c524921b2c6e312cb24741b 2024-11-15T11:52:48,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4ee27aceee964651a3a75f927751a08d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4ee27aceee964651a3a75f927751a08d 2024-11-15T11:52:48,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/40bb148e95cf45f9a171e4619aaae620 2024-11-15T11:52:48,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e063660cf1941e9bc29ca68908747f9 2024-11-15T11:52:48,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/96c98667238c4f3fa796c6aabd13a513 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/96c98667238c4f3fa796c6aabd13a513 2024-11-15T11:52:48,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/e6c6036c94c24a36b2ff8e7590cafd26 2024-11-15T11:52:48,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/71daf825990243a3a6ed035bab8ddc7f 2024-11-15T11:52:48,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/0e5195ba4c79441795ad2272e1d5701a 2024-11-15T11:52:48,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/97503732280e422aa8a9db5ab8e065d9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/97503732280e422aa8a9db5ab8e065d9 2024-11-15T11:52:48,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/07c69275394c41328a7daef88ec0cd7d 2024-11-15T11:52:48,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/314218d04cc44f3c98eb994139782807 2024-11-15T11:52:48,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/2358e5b387ec4f80ad38f72549947911 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/2358e5b387ec4f80ad38f72549947911 2024-11-15T11:52:48,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/761db077ac9c4c95be29465142c9a914 2024-11-15T11:52:48,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/5cf52f95485445cab77160397dc97fec 2024-11-15T11:52:48,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/b9a0014628ee47cb924debe427b37f34 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/b9a0014628ee47cb924debe427b37f34 2024-11-15T11:52:48,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/c4c7872aa32b4218bed3ff50b12e0c19 2024-11-15T11:52:48,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/888e5bb4ba3746be8665ffae2a11389d 2024-11-15T11:52:48,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/35d7ca9f214744578f036fbcf4ce3fb3 2024-11-15T11:52:48,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/d1f4049373794d84ba9a15d4f4247663 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/d1f4049373794d84ba9a15d4f4247663 2024-11-15T11:52:48,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/258bd05fd95242b6958080a947c2d76f 2024-11-15T11:52:48,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f1922a2ca92d45aa9cddace1a3253f0b 2024-11-15T11:52:48,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/8fc2bd0e5d294567bdabbf985c4f0821 2024-11-15T11:52:48,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa58980beb33489a87998fdb40fbf831, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/05cf72936176495099500cee1c1831ca, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/774f0264a8bf4795bbcf2773ec12ce31, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/7278660eb6124a8fbfbce257593678a8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/f040ffb1c38b4554a2be18e44d9b1313, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1ec99f0abc084a1eb3b850d102b0b8cf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/367a9548c67f4c01ad1f4ef654768fc0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2de5a1be47334ee7b29a0a96e6f0c942, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8d8338a0dd6d4caaa2096daf73db2b97, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140] to archive 2024-11-15T11:52:48,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:52:48,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/b25d28827908406bbf81d8d30fdd27a3 2024-11-15T11:52:48,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/e7eb0a2c12ad4f1d9846f833bb725b59 2024-11-15T11:52:48,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa58980beb33489a87998fdb40fbf831 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa58980beb33489a87998fdb40fbf831 2024-11-15T11:52:48,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2ebb7eb3231540ef937d5115c82215bd 2024-11-15T11:52:48,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/df9b16503f1c4d01917a4d2b0ccfeb2a 2024-11-15T11:52:48,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/05cf72936176495099500cee1c1831ca to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/05cf72936176495099500cee1c1831ca 2024-11-15T11:52:48,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8f1bf0d582dd445ea6c4426d54718b2b 2024-11-15T11:52:48,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/333b535311684ad7877bf6ba11ab47c0 2024-11-15T11:52:48,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/080eb44123d04c2491f93698352d7b77 2024-11-15T11:52:48,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/774f0264a8bf4795bbcf2773ec12ce31 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/774f0264a8bf4795bbcf2773ec12ce31 2024-11-15T11:52:48,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/30eb126323c84f888353bf61cc892635 2024-11-15T11:52:48,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1afb2ed341844c1d910de39ffcdc105e 2024-11-15T11:52:48,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/7278660eb6124a8fbfbce257593678a8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/7278660eb6124a8fbfbce257593678a8 2024-11-15T11:52:48,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/ba3d3e66fce84fc48b6a994d35f53334 2024-11-15T11:52:48,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5fc341210d9f4aa3b8c5642e662bb2b9 2024-11-15T11:52:48,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/f040ffb1c38b4554a2be18e44d9b1313 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/f040ffb1c38b4554a2be18e44d9b1313 2024-11-15T11:52:48,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c9a1d992a69a4d919623ac1fd128e872 2024-11-15T11:52:48,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c1915ee5c7ad45a6b71a8f32b1a17632 2024-11-15T11:52:48,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/3d38c2e748e64653aab252576caf3811 2024-11-15T11:52:48,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1ec99f0abc084a1eb3b850d102b0b8cf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/1ec99f0abc084a1eb3b850d102b0b8cf 2024-11-15T11:52:48,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/994fa96d3bd54ab3be7652d4660910d0 2024-11-15T11:52:48,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9e8d60a6404b4915990431ecacf8ede0 2024-11-15T11:52:48,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/367a9548c67f4c01ad1f4ef654768fc0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/367a9548c67f4c01ad1f4ef654768fc0 2024-11-15T11:52:48,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/aa7f2978905246088ef65c633ae43e7d 2024-11-15T11:52:48,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/04bdfcfd41f94d9795d6b0dfd51e15c6 2024-11-15T11:52:48,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2de5a1be47334ee7b29a0a96e6f0c942 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2de5a1be47334ee7b29a0a96e6f0c942 2024-11-15T11:52:48,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/2a34222cd317451ab91d54b782cf3196 2024-11-15T11:52:48,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/db37a11b8b894072af30cf9a525a8641 2024-11-15T11:52:48,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/400731be96dd44cd9c050673e919abc7 2024-11-15T11:52:48,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8d8338a0dd6d4caaa2096daf73db2b97 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/8d8338a0dd6d4caaa2096daf73db2b97 2024-11-15T11:52:48,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/35b42d454c714053bad319cff64031ee 2024-11-15T11:52:48,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/d1ff65b67e5041139bab28cda9cbbe1a 2024-11-15T11:52:48,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/c77ef0bf85a24109a55355e6c3645140 2024-11-15T11:52:48,059 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/recovered.edits/635.seqid, newMaxSeqId=635, maxSeqId=1 2024-11-15T11:52:48,060 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. 2024-11-15T11:52:48,060 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 962de2b7258c31da99a4c1635a09f927: Waiting for close lock at 1731671566420Running coprocessor pre-close hooks at 1731671566420Disabling compacts and flushes for region at 1731671566420Disabling writes for close at 1731671566697 (+277 ms)Obtaining lock to block concurrent updates at 1731671566697Preparing flush snapshotting stores in 962de2b7258c31da99a4c1635a09f927 at 1731671566697Finished memstore snapshotting TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731671566697Flushing stores of TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927. at 1731671566698 (+1 ms)Flushing 962de2b7258c31da99a4c1635a09f927/A: creating writer at 1731671566698Flushing 962de2b7258c31da99a4c1635a09f927/A: appending metadata at 1731671566699 (+1 ms)Flushing 962de2b7258c31da99a4c1635a09f927/A: closing flushed file at 1731671566699Flushing 962de2b7258c31da99a4c1635a09f927/B: creating writer at 1731671567107 (+408 ms)Flushing 962de2b7258c31da99a4c1635a09f927/B: appending metadata at 1731671567109 (+2 ms)Flushing 962de2b7258c31da99a4c1635a09f927/B: closing flushed file at 1731671567109Flushing 962de2b7258c31da99a4c1635a09f927/C: creating writer at 1731671567521 (+412 ms)Flushing 962de2b7258c31da99a4c1635a09f927/C: appending metadata at 1731671567523 (+2 ms)Flushing 962de2b7258c31da99a4c1635a09f927/C: closing flushed file at 1731671567523Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6e6ac1: reopening flushed file at 1731671567937 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e586dc: reopening flushed file at 1731671567943 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1988f6de: reopening flushed file at 1731671567948 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 962de2b7258c31da99a4c1635a09f927 in 1257ms, sequenceid=632, compaction requested=true at 1731671567954 (+6 ms)Writing region close event to WAL at 1731671568057 (+103 ms)Running coprocessor post-close hooks at 1731671568060 (+3 ms)Closed at 1731671568060 2024-11-15T11:52:48,061 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:48,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=962de2b7258c31da99a4c1635a09f927, regionState=CLOSED 2024-11-15T11:52:48,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:48,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T11:52:48,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 962de2b7258c31da99a4c1635a09f927, server=7adf9b3d9d04,44479,1731671471201 in 1.7970 sec 2024-11-15T11:52:48,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-15T11:52:48,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=962de2b7258c31da99a4c1635a09f927, UNASSIGN in 1.8010 sec 2024-11-15T11:52:48,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T11:52:48,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8050 sec 2024-11-15T11:52:48,068 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671568068"}]},"ts":"1731671568068"} 2024-11-15T11:52:48,069 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:52:48,069 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:52:48,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8130 sec 2024-11-15T11:52:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-15T11:52:48,401 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:48,402 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:52:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,404 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-15T11:52:48,405 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,408 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:48,410 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/recovered.edits] 2024-11-15T11:52:48,412 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/26a49000136746c68f1808acb4c772b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/26a49000136746c68f1808acb4c772b5 2024-11-15T11:52:48,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/baedfaf94fea4dd18c3f22523643ad3b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/baedfaf94fea4dd18c3f22523643ad3b 2024-11-15T11:52:48,415 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bd907c72258c44519201179059d5b022 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/bd907c72258c44519201179059d5b022 2024-11-15T11:52:48,416 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/de937f8d9fad487f8cb64358a8b563d1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/de937f8d9fad487f8cb64358a8b563d1 2024-11-15T11:52:48,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/df8e35fe82a24687b788dc74c8ed0b08 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/A/df8e35fe82a24687b788dc74c8ed0b08 2024-11-15T11:52:48,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/302915cd2dd6411dbe20b9792a99dd95 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/302915cd2dd6411dbe20b9792a99dd95 2024-11-15T11:52:48,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/458a718d5ae74fbe9394379a5c722b06 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/458a718d5ae74fbe9394379a5c722b06 2024-11-15T11:52:48,422 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4a9147688d8f4f0f98d551d554bc02ed to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/4a9147688d8f4f0f98d551d554bc02ed 2024-11-15T11:52:48,424 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/934db9e184f6421bb91be5d180d7bda4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/934db9e184f6421bb91be5d180d7bda4 2024-11-15T11:52:48,425 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f0806c4ced854ef3b11acf2468bb7d9e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/B/f0806c4ced854ef3b11acf2468bb7d9e 2024-11-15T11:52:48,427 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/342fac1048374f4facf2f3bf29652d95 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/342fac1048374f4facf2f3bf29652d95 2024-11-15T11:52:48,429 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/4b0f93e35044454c97cdefe4ee8378da to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/4b0f93e35044454c97cdefe4ee8378da 2024-11-15T11:52:48,430 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5a3a6679ed024885ae56b328c373f15f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/5a3a6679ed024885ae56b328c373f15f 2024-11-15T11:52:48,431 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/871ee35c2c704a30b35e7987875b2457 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/871ee35c2c704a30b35e7987875b2457 2024-11-15T11:52:48,432 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9eb79c269eed4a2492123f85ac1ee850 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/C/9eb79c269eed4a2492123f85ac1ee850 2024-11-15T11:52:48,435 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/recovered.edits/635.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927/recovered.edits/635.seqid 2024-11-15T11:52:48,435 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/962de2b7258c31da99a4c1635a09f927 2024-11-15T11:52:48,435 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:52:48,437 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,439 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:52:48,441 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:52:48,442 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,442 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:52:48,442 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671568442"}]},"ts":"9223372036854775807"} 2024-11-15T11:52:48,444 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:52:48,444 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 962de2b7258c31da99a4c1635a09f927, NAME => 'TestAcidGuarantees,,1731671543395.962de2b7258c31da99a4c1635a09f927.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:52:48,444 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:52:48,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671568444"}]},"ts":"9223372036854775807"} 2024-11-15T11:52:48,446 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:52:48,447 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-15T11:52:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-15T11:52:48,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:52:48,513 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:48,522 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-160368578_22 at /127.0.0.1:59896 [Waiting for operation #405] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_231585442_22 at /127.0.0.1:33558 [Waiting for operation #314] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=485 (was 393) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10209 (was 10344) 2024-11-15T11:52:48,534 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=485, ProcessCount=11, AvailableMemoryMB=10209 2024-11-15T11:52:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:52:48,536 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:52:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:48,538 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:52:48,539 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:48,539 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-15T11:52:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:48,540 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:52:48,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742188_1364 (size=1061) 2024-11-15T11:52:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:48,947 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:52:48,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742189_1365 (size=53) 2024-11-15T11:52:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:49,355 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:52:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 050c281702ae24e57ecff67ffc93194a, disabling compactions & flushes 2024-11-15T11:52:49,356 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. after waiting 0 ms 2024-11-15T11:52:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,356 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,356 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 050c281702ae24e57ecff67ffc93194a: Waiting for close lock at 1731671569356Disabling compacts and flushes for region at 1731671569356Disabling writes for close at 1731671569356Writing region close event to WAL at 1731671569356Closed at 1731671569356 2024-11-15T11:52:49,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:52:49,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671569357"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671569357"}]},"ts":"1731671569357"} 2024-11-15T11:52:49,359 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:52:49,359 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:52:49,360 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671569359"}]},"ts":"1731671569359"} 2024-11-15T11:52:49,362 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:52:49,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, ASSIGN}] 2024-11-15T11:52:49,364 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, ASSIGN 2024-11-15T11:52:49,366 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:52:49,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:49,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, ASSIGN because future has completed 2024-11-15T11:52:49,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:49,672 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:52:49,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:52:49,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,675 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,676 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:49,676 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName A 2024-11-15T11:52:49,676 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:49,676 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:49,676 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,677 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:49,677 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName B 2024-11-15T11:52:49,677 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:49,678 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:49,678 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,679 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:49,679 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName C 2024-11-15T11:52:49,679 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:49,679 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:49,679 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,679 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,680 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,680 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:49,681 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,681 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,681 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:52:49,682 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,683 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:52:49,684 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 050c281702ae24e57ecff67ffc93194a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67332607, jitterRate=0.003334030508995056}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:52:49,684 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:49,684 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 050c281702ae24e57ecff67ffc93194a: Running coprocessor pre-open hook at 1731671569673Writing region info on filesystem at 1731671569673Initializing all the Stores at 1731671569674 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671569674Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671569674Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671569674Cleaning up temporary data from old regions at 1731671569681 (+7 ms)Running coprocessor post-open hooks at 1731671569684 (+3 ms)Region opened successfully at 1731671569684 2024-11-15T11:52:49,685 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., pid=95, masterSystemTime=1731671569670 2024-11-15T11:52:49,687 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,687 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:49,687 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:49,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:49,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-15T11:52:49,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 in 171 msec 2024-11-15T11:52:49,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-15T11:52:49,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, ASSIGN in 328 msec 2024-11-15T11:52:49,692 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:52:49,693 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671569692"}]},"ts":"1731671569692"} 2024-11-15T11:52:49,694 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:52:49,695 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:52:49,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-15T11:52:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-15T11:52:50,691 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:50,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:50,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:50,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:50,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb0d622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:50,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:50,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:50,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed175f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:50,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:50,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:50,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:50,699 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:52:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:52:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43ba725e 2024-11-15T11:52:50,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:52:50,702 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:52:50,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:52:50,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:52:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T11:52:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742190_1366 (size=1097) 2024-11-15T11:52:51,119 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T11:52:51,119 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T11:52:51,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:52:51,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE}] 2024-11-15T11:52:51,124 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE 2024-11-15T11:52:51,125 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE because future has completed 2024-11-15T11:52:51,127 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:52:51,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:51,279 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,279 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:52:51,279 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 050c281702ae24e57ecff67ffc93194a, disabling compactions & flushes 2024-11-15T11:52:51,279 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,279 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,279 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. after waiting 0 ms 2024-11-15T11:52:51,279 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,283 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T11:52:51,284 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,284 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 050c281702ae24e57ecff67ffc93194a: Waiting for close lock at 1731671571279Running coprocessor pre-close hooks at 1731671571279Disabling compacts and flushes for region at 1731671571279Disabling writes for close at 1731671571279Writing region close event to WAL at 1731671571280 (+1 ms)Running coprocessor post-close hooks at 1731671571284 (+4 ms)Closed at 1731671571284 2024-11-15T11:52:51,284 WARN [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 050c281702ae24e57ecff67ffc93194a to self. 2024-11-15T11:52:51,286 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,287 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=CLOSED 2024-11-15T11:52:51,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:51,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-15T11:52:51,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 in 162 msec 2024-11-15T11:52:51,292 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE; state=CLOSED, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=true 2024-11-15T11:52:51,442 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE because future has completed 2024-11-15T11:52:51,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:52:51,599 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,599 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:52:51,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:52:51,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,601 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,602 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:51,602 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName A 2024-11-15T11:52:51,603 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:51,604 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:51,604 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,605 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:51,605 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName B 2024-11-15T11:52:51,605 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:51,606 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:51,606 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,606 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:52:51,607 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 050c281702ae24e57ecff67ffc93194a columnFamilyName C 2024-11-15T11:52:51,607 DEBUG [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:51,607 INFO [StoreOpener-050c281702ae24e57ecff67ffc93194a-1 {}] regionserver.HStore(327): Store=050c281702ae24e57ecff67ffc93194a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:52:51,607 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,607 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,608 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,609 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,610 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,611 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,611 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:52:51,613 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,613 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 050c281702ae24e57ecff67ffc93194a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68753621, jitterRate=0.02450878918170929}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:52:51,614 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,615 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 050c281702ae24e57ecff67ffc93194a: Running coprocessor pre-open hook at 1731671571600Writing region info on filesystem at 1731671571600Initializing all the Stores at 1731671571601 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671571601Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671571601Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671571601Cleaning up temporary data from old regions at 1731671571611 (+10 ms)Running coprocessor post-open hooks at 1731671571614 (+3 ms)Region opened successfully at 1731671571614 2024-11-15T11:52:51,616 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., pid=100, masterSystemTime=1731671571596 2024-11-15T11:52:51,619 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,619 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=OPEN, openSeqNum=5, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:52:51,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-15T11:52:51,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 in 177 msec 2024-11-15T11:52:51,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T11:52:51,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, REOPEN/MOVE in 499 msec 2024-11-15T11:52:51,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-15T11:52:51,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-15T11:52:51,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 924 msec 2024-11-15T11:52:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-15T11:52:51,631 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:51,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,633 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342af2ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,634 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2067078b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,637 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9321da6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368db88f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,640 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4deb30d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1067630b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,644 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4830f0aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,645 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7d197e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51dbc466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c18d8e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,651 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633e49cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,653 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34560a31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,655 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216e1587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467bd178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302e6f1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,661 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19010f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb7df86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e159d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:52:51,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:52:51,672 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:52:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:52:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:52:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a954be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:52:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:52:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:52:51,673 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:52:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ff7728, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:52:51,675 DEBUG [Thread-1666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,679 DEBUG [Thread-1668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,687 DEBUG [Thread-1655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,698 DEBUG [Thread-1661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,702 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:51,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:51,727 DEBUG [Thread-1672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T11:52:51,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,731 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:51,733 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:51,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:51,741 DEBUG [Thread-1659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,742 DEBUG [Thread-1653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,748 DEBUG [Thread-1657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e597dd706304fba8dda800c9f5e275e_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671571704/Put/seqid=0 2024-11-15T11:52:51,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731671571743, max timestamp=1731671571754Num uniques 859; 2024-11-15T11:52:51,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731671571743, max timestamp=1731671571754Num uniques 859; 2024-11-15T11:52:51,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:51,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:51,759 DEBUG [Thread-1670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:52:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:52:51,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671571743, max timestamp=1731671571754Num uniques 859; 2024-11-15T11:52:51,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:52:51,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:52:51,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:52:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:52:51,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742191_1367 (size=9714) 2024-11-15T11:52:51,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:51,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e597dd706304fba8dda800c9f5e275e_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e597dd706304fba8dda800c9f5e275e_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:51,801 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/e8f6311bcf0b46848fb3f372d131a496, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:51,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/e8f6311bcf0b46848fb3f372d131a496 is 175, key is test_row_0/A:col10/1731671571704/Put/seqid=0 2024-11-15T11:52:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671631787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671631789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671631797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671631804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671631805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:51,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742192_1368 (size=22361) 2024-11-15T11:52:51,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:51,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:51,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:51,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671631910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671631911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671631930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671631930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671631930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671632120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671632130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671632151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671632151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671632151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,249 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/e8f6311bcf0b46848fb3f372d131a496 2024-11-15T11:52:52,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c1092555ec374e098e6159c4a8459d34 is 50, key is test_row_0/B:col10/1731671571704/Put/seqid=0 2024-11-15T11:52:52,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742193_1369 (size=9657) 2024-11-15T11:52:52,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c1092555ec374e098e6159c4a8459d34 2024-11-15T11:52:52,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/ca4a8bf1be2444c39e2a6a541001742d is 50, key is test_row_0/C:col10/1731671571704/Put/seqid=0 2024-11-15T11:52:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:52,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742194_1370 (size=9657) 2024-11-15T11:52:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671632431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671632451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671632460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671632460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671632460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:52,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/ca4a8bf1be2444c39e2a6a541001742d 2024-11-15T11:52:52,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/e8f6311bcf0b46848fb3f372d131a496 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496 2024-11-15T11:52:52,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496, entries=100, sequenceid=14, filesize=21.8 K 2024-11-15T11:52:52,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c1092555ec374e098e6159c4a8459d34 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34 2024-11-15T11:52:52,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34, entries=100, sequenceid=14, filesize=9.4 K 2024-11-15T11:52:52,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/ca4a8bf1be2444c39e2a6a541001742d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d 2024-11-15T11:52:52,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d, entries=100, sequenceid=14, filesize=9.4 K 2024-11-15T11:52:52,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 050c281702ae24e57ecff67ffc93194a in 1078ms, sequenceid=14, compaction requested=false 2024-11-15T11:52:52,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:52,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:52,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:52,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:52,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dda9d2e31c59499180bd4f7bdef48a24_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671571773/Put/seqid=0 2024-11-15T11:52:52,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742195_1371 (size=12154) 2024-11-15T11:52:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:52,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671632971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671632976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671632977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671632978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671632979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671633090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671633091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671633101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671633101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:53,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dda9d2e31c59499180bd4f7bdef48a24_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dda9d2e31c59499180bd4f7bdef48a24_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:53,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b872eb0924e945b98320527d5c1231e3, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b872eb0924e945b98320527d5c1231e3 is 175, key is test_row_0/A:col10/1731671571773/Put/seqid=0 2024-11-15T11:52:53,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742196_1372 (size=30955) 2024-11-15T11:52:53,231 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b872eb0924e945b98320527d5c1231e3 2024-11-15T11:52:53,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/e3d9389903e44998bb6272929010978a is 50, key is test_row_0/B:col10/1731671571773/Put/seqid=0 2024-11-15T11:52:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742197_1373 (size=12001) 2024-11-15T11:52:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671633300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671633312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671633324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671633324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,592 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:52:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671633610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671633620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,649 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/e3d9389903e44998bb6272929010978a 2024-11-15T11:52:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671633641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671633641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/8fa2830804ac455c9e2feb56f5ff1f4e is 50, key is test_row_0/C:col10/1731671571773/Put/seqid=0 2024-11-15T11:52:53,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742198_1374 (size=12001) 2024-11-15T11:52:53,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/8fa2830804ac455c9e2feb56f5ff1f4e 2024-11-15T11:52:53,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b872eb0924e945b98320527d5c1231e3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3 2024-11-15T11:52:53,697 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3, entries=150, sequenceid=49, filesize=30.2 K 2024-11-15T11:52:53,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/e3d9389903e44998bb6272929010978a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a 2024-11-15T11:52:53,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T11:52:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/8fa2830804ac455c9e2feb56f5ff1f4e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e 2024-11-15T11:52:53,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T11:52:53,710 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 050c281702ae24e57ecff67ffc93194a in 902ms, sequenceid=49, compaction requested=false 2024-11-15T11:52:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:53,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T11:52:53,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T11:52:53,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T11:52:53,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9790 sec 2024-11-15T11:52:53,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9880 sec 2024-11-15T11:52:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T11:52:53,881 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:53,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T11:52:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:53,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:53,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:53,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:53,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:52:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:53,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:54,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152074042e57ff4fa0bda19fdab7df771a_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:54,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742199_1375 (size=14594) 2024-11-15T11:52:54,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:54,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152074042e57ff4fa0bda19fdab7df771a_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152074042e57ff4fa0bda19fdab7df771a_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:54,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/46426b33ce0b41fea725c2b4629ee5fd, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:54,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/46426b33ce0b41fea725c2b4629ee5fd is 175, key is test_row_0/A:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742200_1376 (size=39549) 2024-11-15T11:52:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671574107, max timestamp=1731671574138Num uniques 859; 2024-11-15T11:52:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671574107, max timestamp=1731671574138Num uniques 859; 2024-11-15T11:52:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:54,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:54,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671574107, max timestamp=1731671574138Num uniques 859; 2024-11-15T11:52:54,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671634177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671634188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671634192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671634195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671634199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671634301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671634301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671634310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671634311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671634326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,468 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/46426b33ce0b41fea725c2b4629ee5fd 2024-11-15T11:52:54,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/aa153600246442f5879f63de0d9a5d48 is 50, key is test_row_0/B:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:54,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742201_1377 (size=12001) 2024-11-15T11:52:54,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:54,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671634520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671634520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671634531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671634531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671634550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:54,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671634840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671634841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671634853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671634853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671634870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:54,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/aa153600246442f5879f63de0d9a5d48 2024-11-15T11:52:54,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3b6f8966ad51495998e84c337f293c8f is 50, key is test_row_0/C:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742202_1378 (size=12001) 2024-11-15T11:52:54,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3b6f8966ad51495998e84c337f293c8f 2024-11-15T11:52:54,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/46426b33ce0b41fea725c2b4629ee5fd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd 2024-11-15T11:52:54,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd, entries=200, sequenceid=60, filesize=38.6 K 2024-11-15T11:52:54,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/aa153600246442f5879f63de0d9a5d48 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48 2024-11-15T11:52:54,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T11:52:54,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3b6f8966ad51495998e84c337f293c8f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f 2024-11-15T11:52:54,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T11:52:54,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 050c281702ae24e57ecff67ffc93194a in 947ms, sequenceid=60, compaction requested=true 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:54,941 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:54,941 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:54,944 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:54,945 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:52:54,945 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,945 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=90.7 K 2024-11-15T11:52:54,945 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,945 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd] 2024-11-15T11:52:54,945 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:54,945 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:52:54,945 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,945 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=32.9 K 2024-11-15T11:52:54,946 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c1092555ec374e098e6159c4a8459d34, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731671571704 2024-11-15T11:52:54,946 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f6311bcf0b46848fb3f372d131a496, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731671571704 2024-11-15T11:52:54,946 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d9389903e44998bb6272929010978a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671571773 2024-11-15T11:52:54,946 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting aa153600246442f5879f63de0d9a5d48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:54,946 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b872eb0924e945b98320527d5c1231e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671571773 2024-11-15T11:52:54,948 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46426b33ce0b41fea725c2b4629ee5fd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:54,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:54,965 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:54,972 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:54,973 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b599658e582049d5aafa1848b257f9f1 is 50, key is test_row_0/B:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:54,974 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:54,981 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411158d2be6eb454a43dea39f956fe61045fe_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:54,983 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411158d2be6eb454a43dea39f956fe61045fe_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:54,983 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158d2be6eb454a43dea39f956fe61045fe_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:54,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ee3d62af12c4a25aeeabca7f530d4c0_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671574163/Put/seqid=0 2024-11-15T11:52:54,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742203_1379 (size=12104) 2024-11-15T11:52:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742204_1380 (size=4469) 2024-11-15T11:52:55,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742205_1381 (size=12154) 2024-11-15T11:52:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:55,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:55,401 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:55,402 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/1e698a937d56468f8c818759e2b86d78 is 175, key is test_row_0/A:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:55,403 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b599658e582049d5aafa1848b257f9f1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b599658e582049d5aafa1848b257f9f1 2024-11-15T11:52:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671635390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671635393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:55,412 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into b599658e582049d5aafa1848b257f9f1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:55,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:55,412 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=13, startTime=1731671574941; duration=0sec 2024-11-15T11:52:55,412 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:55,413 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:52:55,413 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:55,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:55,414 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:52:55,414 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ee3d62af12c4a25aeeabca7f530d4c0_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ee3d62af12c4a25aeeabca7f530d4c0_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:55,414 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:55,415 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=32.9 K 2024-11-15T11:52:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0bd3eb5b838f459eab93c77566b2b30c, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0bd3eb5b838f459eab93c77566b2b30c is 175, key is test_row_0/A:col10/1731671574163/Put/seqid=0 2024-11-15T11:52:55,420 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4a8bf1be2444c39e2a6a541001742d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731671571704 2024-11-15T11:52:55,420 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fa2830804ac455c9e2feb56f5ff1f4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671571773 2024-11-15T11:52:55,421 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6f8966ad51495998e84c337f293c8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:55,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671635402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671635404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671635406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742206_1382 (size=31058) 2024-11-15T11:52:55,444 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:55,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/73ff1e18b9ec416c912ee2b7135c7780 is 50, key is test_row_0/C:col10/1731671572975/Put/seqid=0 2024-11-15T11:52:55,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742207_1383 (size=30955) 2024-11-15T11:52:55,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742208_1384 (size=12104) 2024-11-15T11:52:55,488 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/73ff1e18b9ec416c912ee2b7135c7780 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73ff1e18b9ec416c912ee2b7135c7780 2024-11-15T11:52:55,492 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into 73ff1e18b9ec416c912ee2b7135c7780(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:55,492 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:55,493 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=13, startTime=1731671574941; duration=0sec 2024-11-15T11:52:55,493 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:55,493 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:52:55,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671635510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671635531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671635531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671635531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671635731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671635750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671635750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671635750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:55,848 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/1e698a937d56468f8c818759e2b86d78 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78 2024-11-15T11:52:55,855 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into 1e698a937d56468f8c818759e2b86d78(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:55,855 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:55,856 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=13, startTime=1731671574941; duration=0sec 2024-11-15T11:52:55,856 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:55,856 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:52:55,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0bd3eb5b838f459eab93c77566b2b30c 2024-11-15T11:52:55,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c890a1738115483fbd8974c1df9dc97a is 50, key is test_row_0/B:col10/1731671574163/Put/seqid=0 2024-11-15T11:52:55,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742209_1385 (size=12001) 2024-11-15T11:52:55,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c890a1738115483fbd8974c1df9dc97a 2024-11-15T11:52:55,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/267df7edc96d4b899e5900604cd163f7 is 50, key is test_row_0/C:col10/1731671574163/Put/seqid=0 2024-11-15T11:52:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742210_1386 (size=12001) 2024-11-15T11:52:56,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671636050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671636060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671636071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671636071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,312 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/267df7edc96d4b899e5900604cd163f7 2024-11-15T11:52:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0bd3eb5b838f459eab93c77566b2b30c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c 2024-11-15T11:52:56,323 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T11:52:56,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c890a1738115483fbd8974c1df9dc97a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a 2024-11-15T11:52:56,330 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T11:52:56,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/267df7edc96d4b899e5900604cd163f7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7 2024-11-15T11:52:56,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T11:52:56,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 050c281702ae24e57ecff67ffc93194a in 1387ms, sequenceid=95, compaction requested=false 2024-11-15T11:52:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T11:52:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T11:52:56,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T11:52:56,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4710 sec 2024-11-15T11:52:56,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.4760 sec 2024-11-15T11:52:56,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:56,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:56,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:56,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115880f38a6b8294cc0a8c1e501ca7cacb4_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:56,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742211_1387 (size=14594) 2024-11-15T11:52:56,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671636531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671636570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671636570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671636590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671636590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671636640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:56,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:56,827 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115880f38a6b8294cc0a8c1e501ca7cacb4_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115880f38a6b8294cc0a8c1e501ca7cacb4_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:56,828 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/a5c6df296b734765b6baf2fba344f5ee, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:56,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/a5c6df296b734765b6baf2fba344f5ee is 175, key is test_row_0/A:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742212_1388 (size=39549) 2024-11-15T11:52:56,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/a5c6df296b734765b6baf2fba344f5ee 2024-11-15T11:52:56,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b60e857b497942f2b96aeebd1bb12d60 is 50, key is test_row_0/B:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:56,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742213_1389 (size=12001) 2024-11-15T11:52:56,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:56,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671636861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671637180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b60e857b497942f2b96aeebd1bb12d60 2024-11-15T11:52:57,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4d26c967500648f8ac9fe5a72768116f is 50, key is test_row_0/C:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:57,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742214_1390 (size=12001) 2024-11-15T11:52:57,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4d26c967500648f8ac9fe5a72768116f 2024-11-15T11:52:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/a5c6df296b734765b6baf2fba344f5ee as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee 2024-11-15T11:52:57,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee, entries=200, sequenceid=110, filesize=38.6 K 2024-11-15T11:52:57,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b60e857b497942f2b96aeebd1bb12d60 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60 2024-11-15T11:52:57,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T11:52:57,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4d26c967500648f8ac9fe5a72768116f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f 2024-11-15T11:52:57,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f, entries=150, sequenceid=110, filesize=11.7 K 2024-11-15T11:52:57,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 050c281702ae24e57ecff67ffc93194a in 885ms, sequenceid=110, compaction requested=true 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:52:57,298 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:57,298 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:57,298 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:57,298 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:52:57,299 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:57,299 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:57,299 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=99.2 K 2024-11-15T11:52:57,299 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b599658e582049d5aafa1848b257f9f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=35.3 K 2024-11-15T11:52:57,299 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee] 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b599658e582049d5aafa1848b257f9f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e698a937d56468f8c818759e2b86d78, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd3eb5b838f459eab93c77566b2b30c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671574163 2024-11-15T11:52:57,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c890a1738115483fbd8974c1df9dc97a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671574163 2024-11-15T11:52:57,300 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c6df296b734765b6baf2fba344f5ee, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575400 2024-11-15T11:52:57,300 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b60e857b497942f2b96aeebd1bb12d60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575402 2024-11-15T11:52:57,310 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:57,311 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/f7916c6450b943e0a0120ae3dde0e5ce is 50, key is test_row_0/B:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:57,315 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:57,326 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115fbc14090b42f49f0abc1bf9e949df41a_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742215_1391 (size=12207) 2024-11-15T11:52:57,327 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115fbc14090b42f49f0abc1bf9e949df41a_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:57,327 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fbc14090b42f49f0abc1bf9e949df41a_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742216_1392 (size=4469) 2024-11-15T11:52:57,338 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/f7916c6450b943e0a0120ae3dde0e5ce as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f7916c6450b943e0a0120ae3dde0e5ce 2024-11-15T11:52:57,338 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#334 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:57,339 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/af797ff8961d4195997d74cc0cc3b441 is 175, key is test_row_0/A:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:57,354 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into f7916c6450b943e0a0120ae3dde0e5ce(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:57,356 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:57,356 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=13, startTime=1731671577298; duration=0sec 2024-11-15T11:52:57,356 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:52:57,356 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:52:57,356 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:52:57,357 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:52:57,357 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:52:57,357 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:57,358 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73ff1e18b9ec416c912ee2b7135c7780, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=35.3 K 2024-11-15T11:52:57,358 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ff1e18b9ec416c912ee2b7135c7780, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671572975 2024-11-15T11:52:57,361 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 267df7edc96d4b899e5900604cd163f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731671574163 2024-11-15T11:52:57,361 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d26c967500648f8ac9fe5a72768116f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575402 2024-11-15T11:52:57,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742217_1393 (size=31161) 2024-11-15T11:52:57,380 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/af797ff8961d4195997d74cc0cc3b441 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441 2024-11-15T11:52:57,387 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into af797ff8961d4195997d74cc0cc3b441(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:57,387 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:57,387 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=13, startTime=1731671577298; duration=0sec 2024-11-15T11:52:57,387 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:57,387 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:52:57,393 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:52:57,393 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/7e0537000fbc44efb766678a3649e83c is 50, key is test_row_0/C:col10/1731671576411/Put/seqid=0 2024-11-15T11:52:57,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742218_1394 (size=12207) 2024-11-15T11:52:57,433 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/7e0537000fbc44efb766678a3649e83c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/7e0537000fbc44efb766678a3649e83c 2024-11-15T11:52:57,445 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into 7e0537000fbc44efb766678a3649e83c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:52:57,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:57,445 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=13, startTime=1731671577298; duration=0sec 2024-11-15T11:52:57,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:52:57,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:52:57,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:57,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:52:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:57,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111522029777e3214fc0a454e02d0e88e1cb_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671576530/Put/seqid=0 2024-11-15T11:52:57,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742219_1395 (size=14694) 2024-11-15T11:52:57,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:57,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111522029777e3214fc0a454e02d0e88e1cb_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522029777e3214fc0a454e02d0e88e1cb_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:57,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671637625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/99ff507bc6d3428a91b6df7a10e33071, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:57,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/99ff507bc6d3428a91b6df7a10e33071 is 175, key is test_row_0/A:col10/1731671576530/Put/seqid=0 2024-11-15T11:52:57,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742220_1396 (size=39649) 2024-11-15T11:52:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671637626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671637629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671637630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671637691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671637740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671637751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671637751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671637751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671637961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671637961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671637961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671637961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,039 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/99ff507bc6d3428a91b6df7a10e33071 2024-11-15T11:52:58,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T11:52:58,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9885744a5b534c12a6f492eb8b322427 is 50, key is test_row_0/B:col10/1731671576530/Put/seqid=0 2024-11-15T11:52:58,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:52:58,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:52:58,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T11:52:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:52:58,053 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:52:58,054 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:52:58,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:52:58,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742221_1397 (size=12101) 2024-11-15T11:52:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:52:58,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671638271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671638281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671638281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671638281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:52:58,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9885744a5b534c12a6f492eb8b322427 2024-11-15T11:52:58,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/cd0be3447f234351a34a3246422b3211 is 50, key is test_row_0/C:col10/1731671576530/Put/seqid=0 2024-11-15T11:52:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742222_1398 (size=12101) 2024-11-15T11:52:58,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:58,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:52:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671638711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,725 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:52:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671638790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671638801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671638801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671638801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:58,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:58,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:52:58,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/cd0be3447f234351a34a3246422b3211 2024-11-15T11:52:58,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/99ff507bc6d3428a91b6df7a10e33071 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071 2024-11-15T11:52:58,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071, entries=200, sequenceid=134, filesize=38.7 K 2024-11-15T11:52:58,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9885744a5b534c12a6f492eb8b322427 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427 2024-11-15T11:52:58,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T11:52:58,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/cd0be3447f234351a34a3246422b3211 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211 2024-11-15T11:52:58,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T11:52:58,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 050c281702ae24e57ecff67ffc93194a in 1303ms, sequenceid=134, compaction requested=false 2024-11-15T11:52:58,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:52:58,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:52:58,975 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:52:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:52:59,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111583a13b1b60324f8191442d65c20a08e8_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671577629/Put/seqid=0 2024-11-15T11:52:59,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742223_1399 (size=12304) 2024-11-15T11:52:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:52:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:52:59,442 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111583a13b1b60324f8191442d65c20a08e8_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a13b1b60324f8191442d65c20a08e8_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/3b365bffa5ac40c2aa2dd634b03ff74e, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:52:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/3b365bffa5ac40c2aa2dd634b03ff74e is 175, key is test_row_0/A:col10/1731671577629/Put/seqid=0 2024-11-15T11:52:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742224_1400 (size=31105) 2024-11-15T11:52:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:52:59,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:52:59,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/3b365bffa5ac40c2aa2dd634b03ff74e 2024-11-15T11:52:59,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9341fbfb282147ffac121a599f79d68d is 50, key is test_row_0/B:col10/1731671577629/Put/seqid=0 2024-11-15T11:52:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742225_1401 (size=12151) 2024-11-15T11:52:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671639864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671639866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:59,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671579857, max timestamp=1731671579880Num uniques 859; 2024-11-15T11:52:59,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:59,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671579857, max timestamp=1731671579880Num uniques 859; 2024-11-15T11:52:59,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:59,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671579857, max timestamp=1731671579880Num uniques 859; 2024-11-15T11:52:59,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:52:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671639965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:59,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671639976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:52:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:52:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671639990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:52:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:52:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671639990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671640091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671640091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:53:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671640210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671640211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,276 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9341fbfb282147ffac121a599f79d68d 2024-11-15T11:53:00,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/49e64d7a50c74872a854586006bb5aa7 is 50, key is test_row_0/C:col10/1731671577629/Put/seqid=0 2024-11-15T11:53:00,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742226_1402 (size=12151) 2024-11-15T11:53:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671640310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671640311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671640530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671640530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671640631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671640631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:00,686 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/49e64d7a50c74872a854586006bb5aa7 2024-11-15T11:53:00,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/3b365bffa5ac40c2aa2dd634b03ff74e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e 2024-11-15T11:53:00,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e, entries=150, sequenceid=145, filesize=30.4 K 2024-11-15T11:53:00,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/9341fbfb282147ffac121a599f79d68d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d 2024-11-15T11:53:00,700 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T11:53:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/49e64d7a50c74872a854586006bb5aa7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7 2024-11-15T11:53:00,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7, entries=150, sequenceid=145, filesize=11.9 K 2024-11-15T11:53:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 050c281702ae24e57ecff67ffc93194a in 1731ms, sequenceid=145, compaction requested=true 2024-11-15T11:53:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T11:53:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T11:53:00,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T11:53:00,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6540 sec 2024-11-15T11:53:00,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.6620 sec 2024-11-15T11:53:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:00,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:00,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:00,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111535ff65fe2ecb4c37af4b4a62a1feb385_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:00,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742227_1403 (size=14794) 2024-11-15T11:53:00,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671640879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:00,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671641000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671641051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671641051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671641151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671641151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:01,187 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111535ff65fe2ecb4c37af4b4a62a1feb385_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111535ff65fe2ecb4c37af4b4a62a1feb385_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:01,188 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/ca98db6bbaef401fa8024b872e90ac8e, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:01,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/ca98db6bbaef401fa8024b872e90ac8e is 175, key is test_row_0/A:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:01,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742228_1404 (size=39749) 2024-11-15T11:53:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671641220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671641541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:01,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:01,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/ca98db6bbaef401fa8024b872e90ac8e 2024-11-15T11:53:01,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/a96299d09f21487b8a207048c3d063c2 is 50, key is test_row_0/B:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:01,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742229_1405 (size=12151) 2024-11-15T11:53:02,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/a96299d09f21487b8a207048c3d063c2 2024-11-15T11:53:02,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/87fedb39f81249eea18426b8f7ebb444 is 50, key is test_row_0/C:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:02,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742230_1406 (size=12151) 2024-11-15T11:53:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671642050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:02,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671642070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:02,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671642070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:02,079 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:02,079 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:02,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671642180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:02,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671642180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:02,191 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:02,191 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T11:53:02,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:02,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T11:53:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:02,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:02,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:02,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:02,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T11:53:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:02,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:02,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:02,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/87fedb39f81249eea18426b8f7ebb444 2024-11-15T11:53:02,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/ca98db6bbaef401fa8024b872e90ac8e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e 2024-11-15T11:53:02,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e, entries=200, sequenceid=181, filesize=38.8 K 2024-11-15T11:53:02,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/a96299d09f21487b8a207048c3d063c2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2 2024-11-15T11:53:02,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2, entries=150, sequenceid=181, filesize=11.9 K 2024-11-15T11:53:02,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/87fedb39f81249eea18426b8f7ebb444 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444 2024-11-15T11:53:02,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444, entries=150, sequenceid=181, filesize=11.9 K 2024-11-15T11:53:02,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 050c281702ae24e57ecff67ffc93194a in 1701ms, sequenceid=181, compaction requested=true 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:02,449 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:02,449 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:02,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:53:02,450 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,450 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,450 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f7916c6450b943e0a0120ae3dde0e5ce, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=47.5 K 2024-11-15T11:53:02,450 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=138.3 K 2024-11-15T11:53:02,450 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e] 2024-11-15T11:53:02,450 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f7916c6450b943e0a0120ae3dde0e5ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575402 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting af797ff8961d4195997d74cc0cc3b441, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575402 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9885744a5b534c12a6f492eb8b322427, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731671576512 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ff507bc6d3428a91b6df7a10e33071, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731671576486 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9341fbfb282147ffac121a599f79d68d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671577614 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b365bffa5ac40c2aa2dd634b03ff74e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671577614 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a96299d09f21487b8a207048c3d063c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579963 2024-11-15T11:53:02,451 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca98db6bbaef401fa8024b872e90ac8e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579929 2024-11-15T11:53:02,457 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:02,459 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411158032695ada1b424bbe473609dfd36599_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:02,459 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:02,460 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/26389b4eb2dd49dbb76ce142a87eec56 is 50, key is test_row_0/B:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:02,462 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411158032695ada1b424bbe473609dfd36599_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:02,462 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158032695ada1b424bbe473609dfd36599_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:02,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742231_1407 (size=12493) 2024-11-15T11:53:02,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742232_1408 (size=4469) 2024-11-15T11:53:02,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T11:53:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba6d212295d44e54ab8f2ff635b22427_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671580783/Put/seqid=0 2024-11-15T11:53:02,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742233_1409 (size=12304) 2024-11-15T11:53:02,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:02,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ba6d212295d44e54ab8f2ff635b22427_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba6d212295d44e54ab8f2ff635b22427_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/11fe6a658aa24cbcaffa761046327667, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/11fe6a658aa24cbcaffa761046327667 is 175, key is test_row_0/A:col10/1731671580783/Put/seqid=0 2024-11-15T11:53:02,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742234_1410 (size=31105) 2024-11-15T11:53:02,548 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/11fe6a658aa24cbcaffa761046327667 2024-11-15T11:53:02,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/57a8fefb9e034c58954812569ee1c97a is 50, key is test_row_0/B:col10/1731671580783/Put/seqid=0 2024-11-15T11:53:02,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742235_1411 (size=12151) 2024-11-15T11:53:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:02,868 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:02,868 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/26389b4eb2dd49dbb76ce142a87eec56 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/26389b4eb2dd49dbb76ce142a87eec56 2024-11-15T11:53:02,869 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/160889d1c2f142129750c2502ca10291 is 175, key is test_row_0/A:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:02,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742236_1412 (size=31447) 2024-11-15T11:53:02,874 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into 26389b4eb2dd49dbb76ce142a87eec56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:02,874 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:02,874 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=12, startTime=1731671582449; duration=0sec 2024-11-15T11:53:02,874 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:02,874 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:53:02,874 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:02,875 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:02,875 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:53:02,875 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:02,876 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/7e0537000fbc44efb766678a3649e83c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=47.5 K 2024-11-15T11:53:02,876 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e0537000fbc44efb766678a3649e83c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731671575402 2024-11-15T11:53:02,876 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0be3447f234351a34a3246422b3211, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731671576512 2024-11-15T11:53:02,876 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 49e64d7a50c74872a854586006bb5aa7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731671577614 2024-11-15T11:53:02,877 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 87fedb39f81249eea18426b8f7ebb444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579963 2024-11-15T11:53:02,879 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/160889d1c2f142129750c2502ca10291 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291 2024-11-15T11:53:02,889 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:02,890 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into 160889d1c2f142129750c2502ca10291(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:02,890 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:02,890 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=12, startTime=1731671582449; duration=0sec 2024-11-15T11:53:02,890 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/417f524df2c649ce91eb65d74144f6a6 is 50, key is test_row_0/C:col10/1731671579963/Put/seqid=0 2024-11-15T11:53:02,890 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:02,890 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:53:02,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742237_1413 (size=12493) 2024-11-15T11:53:02,966 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/57a8fefb9e034c58954812569ee1c97a 2024-11-15T11:53:02,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/be571e3ac8314c81b2e0bf95957a2f6e is 50, key is test_row_0/C:col10/1731671580783/Put/seqid=0 2024-11-15T11:53:02,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742238_1414 (size=12151) 2024-11-15T11:53:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:03,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671583117, max timestamp=1731671583199Num uniques 859; 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671583117, max timestamp=1731671583199Num uniques 859; 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671583117, max timestamp=1731671583199Num uniques 859; 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:03,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:03,314 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/417f524df2c649ce91eb65d74144f6a6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/417f524df2c649ce91eb65d74144f6a6 2024-11-15T11:53:03,317 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into 417f524df2c649ce91eb65d74144f6a6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:03,317 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:03,317 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=12, startTime=1731671582449; duration=0sec 2024-11-15T11:53:03,317 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:03,317 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:53:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671643317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:03,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/be571e3ac8314c81b2e0bf95957a2f6e 2024-11-15T11:53:03,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/11fe6a658aa24cbcaffa761046327667 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667 2024-11-15T11:53:03,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667, entries=150, sequenceid=191, filesize=30.4 K 2024-11-15T11:53:03,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/57a8fefb9e034c58954812569ee1c97a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a 2024-11-15T11:53:03,391 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T11:53:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/be571e3ac8314c81b2e0bf95957a2f6e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e 2024-11-15T11:53:03,396 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T11:53:03,396 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 050c281702ae24e57ecff67ffc93194a in 867ms, sequenceid=191, compaction requested=false 2024-11-15T11:53:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:03,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T11:53:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T11:53:03,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T11:53:03,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1740 sec 2024-11-15T11:53:03,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.1790 sec 2024-11-15T11:53:03,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:03,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:03,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115537d7d1a528a43e3ab6846615168842d_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742239_1415 (size=14794) 2024-11-15T11:53:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671643491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:03,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671643611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:03,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671643830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:03,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:03,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:03,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115537d7d1a528a43e3ab6846615168842d_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115537d7d1a528a43e3ab6846615168842d_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:03,854 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/f7f657f4ef6e4711b555f9c33fb45201, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:03,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/f7f657f4ef6e4711b555f9c33fb45201 is 175, key is test_row_0/A:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742240_1416 (size=39749) 2024-11-15T11:53:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671644100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671644100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,106 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,106 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671644150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671644200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,209 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671644211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,217 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 050c281702ae24e57ecff67ffc93194a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/f7f657f4ef6e4711b555f9c33fb45201 2024-11-15T11:53:04,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2a5ae11b89644367972cd32cb698463d is 50, key is test_row_0/B:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:04,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742241_1417 (size=12151) 2024-11-15T11:53:04,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2a5ae11b89644367972cd32cb698463d 2024-11-15T11:53:04,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4bd97d7d280e469bbfcb576dc91b74ae is 50, key is test_row_0/C:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742242_1418 (size=12151) 2024-11-15T11:53:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T11:53:04,371 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:04,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T11:53:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:04,373 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:04,373 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:04,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:04,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T11:53:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:04,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671644660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:04,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T11:53:04,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:04,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:04,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4bd97d7d280e469bbfcb576dc91b74ae 2024-11-15T11:53:04,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/f7f657f4ef6e4711b555f9c33fb45201 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201 2024-11-15T11:53:04,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201, entries=200, sequenceid=232, filesize=38.8 K 2024-11-15T11:53:04,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2a5ae11b89644367972cd32cb698463d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d 2024-11-15T11:53:04,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T11:53:04,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/4bd97d7d280e469bbfcb576dc91b74ae as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae 2024-11-15T11:53:04,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T11:53:04,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 050c281702ae24e57ecff67ffc93194a in 1287ms, sequenceid=232, compaction requested=true 2024-11-15T11:53:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:04,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:04,726 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:04,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:04,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:53:04,728 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,728 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=99.9 K 2024-11-15T11:53:04,728 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,728 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201] 2024-11-15T11:53:04,729 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 160889d1c2f142129750c2502ca10291, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579963 2024-11-15T11:53:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:04,729 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:04,729 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11fe6a658aa24cbcaffa761046327667, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731671580783 2024-11-15T11:53:04,729 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7f657f4ef6e4711b555f9c33fb45201, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:04,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:04,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:53:04,730 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,730 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/26389b4eb2dd49dbb76ce142a87eec56, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=35.9 K 2024-11-15T11:53:04,730 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 26389b4eb2dd49dbb76ce142a87eec56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579963 2024-11-15T11:53:04,731 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a8fefb9e034c58954812569ee1c97a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731671580783 2024-11-15T11:53:04,731 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a5ae11b89644367972cd32cb698463d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:04,739 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:04,741 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:04,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/f0735e5461cc4a63995da2e364264a02 is 50, key is test_row_0/B:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:04,742 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ffe74135dbeb4b778658ac28a2cc956d_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:04,744 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ffe74135dbeb4b778658ac28a2cc956d_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:04,744 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ffe74135dbeb4b778658ac28a2cc956d_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:04,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742243_1419 (size=12595) 2024-11-15T11:53:04,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742244_1420 (size=4469) 2024-11-15T11:53:04,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T11:53:04,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:04,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:04,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:04,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151fae362ea72d45c3a30480bdc7b0d2ad_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671583475/Put/seqid=0 2024-11-15T11:53:04,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742245_1421 (size=12304) 2024-11-15T11:53:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:04,877 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151fae362ea72d45c3a30480bdc7b0d2ad_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151fae362ea72d45c3a30480bdc7b0d2ad_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/fca90e5e7c0f41b19fe899e06e7b88bf, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/fca90e5e7c0f41b19fe899e06e7b88bf is 175, key is test_row_0/A:col10/1731671583475/Put/seqid=0 2024-11-15T11:53:04,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742246_1422 (size=31105) 2024-11-15T11:53:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:05,148 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#354 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:05,148 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0c543a988c42404fb646509ce305a169 is 175, key is test_row_0/A:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:05,150 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/f0735e5461cc4a63995da2e364264a02 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f0735e5461cc4a63995da2e364264a02 2024-11-15T11:53:05,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742247_1423 (size=31549) 2024-11-15T11:53:05,154 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into f0735e5461cc4a63995da2e364264a02(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:05,154 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:05,154 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=13, startTime=1731671584729; duration=0sec 2024-11-15T11:53:05,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:05,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:53:05,155 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:05,157 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:05,158 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:53:05,158 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:05,158 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/417f524df2c649ce91eb65d74144f6a6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=35.9 K 2024-11-15T11:53:05,158 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 417f524df2c649ce91eb65d74144f6a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1731671579963 2024-11-15T11:53:05,159 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting be571e3ac8314c81b2e0bf95957a2f6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731671580783 2024-11-15T11:53:05,159 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/0c543a988c42404fb646509ce305a169 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169 2024-11-15T11:53:05,159 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd97d7d280e469bbfcb576dc91b74ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:05,163 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into 0c543a988c42404fb646509ce305a169(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:05,163 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:05,163 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=13, startTime=1731671584726; duration=0sec 2024-11-15T11:53:05,163 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:05,163 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:53:05,168 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:05,168 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/c6e5f6f0939143c994426a46338dfe24 is 50, key is test_row_0/C:col10/1731671583291/Put/seqid=0 2024-11-15T11:53:05,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742248_1424 (size=12595) 2024-11-15T11:53:05,181 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/c6e5f6f0939143c994426a46338dfe24 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/c6e5f6f0939143c994426a46338dfe24 2024-11-15T11:53:05,185 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into c6e5f6f0939143c994426a46338dfe24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:05,185 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:05,185 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=13, startTime=1731671584729; duration=0sec 2024-11-15T11:53:05,185 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:05,185 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:53:05,283 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/fca90e5e7c0f41b19fe899e06e7b88bf 2024-11-15T11:53:05,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c2ec75e4664e4eb1aecc910b84967bad is 50, key is test_row_0/B:col10/1731671583475/Put/seqid=0 2024-11-15T11:53:05,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742249_1425 (size=12151) 2024-11-15T11:53:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:05,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:05,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c2ec75e4664e4eb1aecc910b84967bad 2024-11-15T11:53:05,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/19e55959ff7e4fd1a86f16ee17b81191 is 50, key is test_row_0/C:col10/1731671583475/Put/seqid=0 2024-11-15T11:53:05,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742250_1426 (size=12151) 2024-11-15T11:53:05,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671585756, max timestamp=1731671585802Num uniques 859; 2024-11-15T11:53:05,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671585756, max timestamp=1731671585802Num uniques 859; 2024-11-15T11:53:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671585756, max timestamp=1731671585802Num uniques 859; 2024-11-15T11:53:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:05,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671645946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671646061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/19e55959ff7e4fd1a86f16ee17b81191 2024-11-15T11:53:06,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/fca90e5e7c0f41b19fe899e06e7b88bf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf 2024-11-15T11:53:06,128 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf, entries=150, sequenceid=241, filesize=30.4 K 2024-11-15T11:53:06,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/c2ec75e4664e4eb1aecc910b84967bad as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad 2024-11-15T11:53:06,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad, entries=150, sequenceid=241, filesize=11.9 K 2024-11-15T11:53:06,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/19e55959ff7e4fd1a86f16ee17b81191 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191 2024-11-15T11:53:06,136 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191, entries=150, sequenceid=241, filesize=11.9 K 2024-11-15T11:53:06,137 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 050c281702ae24e57ecff67ffc93194a in 1306ms, sequenceid=241, compaction requested=false 2024-11-15T11:53:06,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:06,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T11:53:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T11:53:06,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T11:53:06,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-11-15T11:53:06,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.7690 sec 2024-11-15T11:53:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:06,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:06,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:06,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154f6c0648301343468b9821c580e365e9_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:06,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742251_1427 (size=14994) 2024-11-15T11:53:06,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:06,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671646314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671646431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T11:53:06,521 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:06,521 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-15T11:53:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:06,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:06,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:06,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:06,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671646650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:06,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154f6c0648301343468b9821c580e365e9_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154f6c0648301343468b9821c580e365e9_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:06,692 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/221d81c20a544132ad13e78dcb565ced, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:06,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/221d81c20a544132ad13e78dcb565ced is 175, key is test_row_0/A:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742252_1428 (size=39949) 2024-11-15T11:53:06,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:06,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:06,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671646960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:06,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:06,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:06,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/221d81c20a544132ad13e78dcb565ced 2024-11-15T11:53:07,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/00001b298f5642b5a8f8baf619245a3a is 50, key is test_row_0/B:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:07,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742253_1429 (size=12301) 2024-11-15T11:53:07,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/00001b298f5642b5a8f8baf619245a3a 2024-11-15T11:53:07,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/262ddb1a70be4d049a46d411a75bd284 is 50, key is test_row_0/C:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:07,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742254_1430 (size=12301) 2024-11-15T11:53:07,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:07,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:07,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671647480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:07,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/262ddb1a70be4d049a46d411a75bd284 2024-11-15T11:53:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/221d81c20a544132ad13e78dcb565ced as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced 2024-11-15T11:53:07,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced, entries=200, sequenceid=282, filesize=39.0 K 2024-11-15T11:53:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/00001b298f5642b5a8f8baf619245a3a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a 2024-11-15T11:53:07,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T11:53:07,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/262ddb1a70be4d049a46d411a75bd284 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284 2024-11-15T11:53:07,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284, entries=150, sequenceid=282, filesize=12.0 K 2024-11-15T11:53:07,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 050c281702ae24e57ecff67ffc93194a in 1253ms, sequenceid=282, compaction requested=true 2024-11-15T11:53:07,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:07,532 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:07,532 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:07,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:07,533 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:07,533 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:53:07,533 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,533 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f0735e5461cc4a63995da2e364264a02, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=36.2 K 2024-11-15T11:53:07,535 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f0735e5461cc4a63995da2e364264a02, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:07,535 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ec75e4664e4eb1aecc910b84967bad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731671583453 2024-11-15T11:53:07,535 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:07,535 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:53:07,535 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 00001b298f5642b5a8f8baf619245a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:07,536 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,536 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=100.2 K 2024-11-15T11:53:07,536 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,536 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced] 2024-11-15T11:53:07,536 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c543a988c42404fb646509ce305a169, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:07,536 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca90e5e7c0f41b19fe899e06e7b88bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731671583453 2024-11-15T11:53:07,536 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221d81c20a544132ad13e78dcb565ced, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:07,542 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:07,543 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:07,543 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/1b51d5a70ae14fbd93fb36b011db89f8 is 50, key is test_row_0/B:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:07,544 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111515f1e7624f47490398a5c7ac2a59134a_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:07,545 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111515f1e7624f47490398a5c7ac2a59134a_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:07,545 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515f1e7624f47490398a5c7ac2a59134a_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742255_1431 (size=4469) 2024-11-15T11:53:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742256_1432 (size=12847) 2024-11-15T11:53:07,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-15T11:53:07,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:07,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111596a63b4e18754a54b2b321796fb96789_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671586288/Put/seqid=0 2024-11-15T11:53:07,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742257_1433 (size=12454) 2024-11-15T11:53:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:07,950 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#364 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:07,950 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/96aafa77364b4c6394010a573cdf9ec3 is 175, key is test_row_0/A:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:07,957 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/1b51d5a70ae14fbd93fb36b011db89f8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/1b51d5a70ae14fbd93fb36b011db89f8 2024-11-15T11:53:07,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742258_1434 (size=31801) 2024-11-15T11:53:07,964 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into 1b51d5a70ae14fbd93fb36b011db89f8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:07,964 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:07,964 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=13, startTime=1731671587532; duration=0sec 2024-11-15T11:53:07,964 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:07,964 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:53:07,964 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:07,965 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/96aafa77364b4c6394010a573cdf9ec3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3 2024-11-15T11:53:07,965 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:07,965 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:53:07,965 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:07,965 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/c6e5f6f0939143c994426a46338dfe24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=36.2 K 2024-11-15T11:53:07,966 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e5f6f0939143c994426a46338dfe24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731671583291 2024-11-15T11:53:07,967 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 19e55959ff7e4fd1a86f16ee17b81191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731671583453 2024-11-15T11:53:07,967 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 262ddb1a70be4d049a46d411a75bd284, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:07,974 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into 96aafa77364b4c6394010a573cdf9ec3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:07,974 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:07,974 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=13, startTime=1731671587532; duration=0sec 2024-11-15T11:53:07,974 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:07,974 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:53:07,980 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:07,980 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/e4e421af382f4b5185742150b6ae7be3 is 50, key is test_row_0/C:col10/1731671585944/Put/seqid=0 2024-11-15T11:53:07,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742259_1435 (size=12847) 2024-11-15T11:53:07,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:08,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111596a63b4e18754a54b2b321796fb96789_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596a63b4e18754a54b2b321796fb96789_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b2f5a59992ac486a9b400c54c152367a, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b2f5a59992ac486a9b400c54c152367a is 175, key is test_row_0/A:col10/1731671586288/Put/seqid=0 2024-11-15T11:53:08,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742260_1436 (size=31255) 2024-11-15T11:53:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:08,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:08,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671588165, max timestamp=1731671588192Num uniques 859; 2024-11-15T11:53:08,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:08,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671588165, max timestamp=1731671588192Num uniques 859; 2024-11-15T11:53:08,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671588165, max timestamp=1731671588192Num uniques 859; 2024-11-15T11:53:08,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:08,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:08,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671648264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671648265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671648265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671588240, max timestamp=1731671588277Num uniques 1068; 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671588240, max timestamp=1731671588277Num uniques 1068; 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671588240, max timestamp=1731671588277Num uniques 1068; 2024-11-15T11:53:08,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-15T11:53:08,402 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/e4e421af382f4b5185742150b6ae7be3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/e4e421af382f4b5185742150b6ae7be3 2024-11-15T11:53:08,407 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into e4e421af382f4b5185742150b6ae7be3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:08,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:08,407 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=13, startTime=1731671587532; duration=0sec 2024-11-15T11:53:08,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:08,407 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671588379, max timestamp=1731671588407Num uniques 1027; 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671588379, max timestamp=1731671588407Num uniques 1027; 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671588379, max timestamp=1731671588407Num uniques 1027; 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-15T11:53:08,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-15T11:53:08,412 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b2f5a59992ac486a9b400c54c152367a 2024-11-15T11:53:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/341df0572e2f4bf49f2242afe7e05c79 is 50, key is test_row_0/B:col10/1731671586288/Put/seqid=0 2024-11-15T11:53:08,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742261_1437 (size=12301) 2024-11-15T11:53:08,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731671588456, max timestamp=1731671588470Num uniques 1086; 2024-11-15T11:53:08,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T11:53:08,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731671588456, max timestamp=1731671588470Num uniques 1086; 2024-11-15T11:53:08,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T11:53:08,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731671588456, max timestamp=1731671588470Num uniques 1086; 2024-11-15T11:53:08,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-15T11:53:08,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671588505, max timestamp=1731671588521Num uniques 1045; 2024-11-15T11:53:08,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671588505, max timestamp=1731671588521Num uniques 1045; 2024-11-15T11:53:08,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T11:53:08,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T11:53:08,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731671588505, max timestamp=1731671588521Num uniques 1045; 2024-11-15T11:53:08,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-15T11:53:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671648559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671648561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671648562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671648564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671648565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:08,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671648680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671648680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671648680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671648680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671648680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,821 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/341df0572e2f4bf49f2242afe7e05c79 2024-11-15T11:53:08,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a6e0f3e2723c4834a2ca9c71922fa972 is 50, key is test_row_0/C:col10/1731671586288/Put/seqid=0 2024-11-15T11:53:08,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742262_1438 (size=12301) 2024-11-15T11:53:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671648890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:08,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671649201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671649201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671649201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671649201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671649210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a6e0f3e2723c4834a2ca9c71922fa972 2024-11-15T11:53:09,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b2f5a59992ac486a9b400c54c152367a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a 2024-11-15T11:53:09,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a, entries=150, sequenceid=290, filesize=30.5 K 2024-11-15T11:53:09,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/341df0572e2f4bf49f2242afe7e05c79 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79 2024-11-15T11:53:09,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79, entries=150, sequenceid=290, filesize=12.0 K 2024-11-15T11:53:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a6e0f3e2723c4834a2ca9c71922fa972 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972 2024-11-15T11:53:09,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972, entries=150, sequenceid=290, filesize=12.0 K 2024-11-15T11:53:09,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 050c281702ae24e57ecff67ffc93194a in 1655ms, sequenceid=290, compaction requested=false 2024-11-15T11:53:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:09,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-15T11:53:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-15T11:53:09,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T11:53:09,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7250 sec 2024-11-15T11:53:09,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.7290 sec 2024-11-15T11:53:09,573 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:53:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:09,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=161.02 KB heapSize=457.31 KB 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731671588549, max timestamp=1731671589719Num uniques 1104; 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731671588549, max timestamp=1731671589719Num uniques 1104; 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731671588549, max timestamp=1731671589719Num uniques 1104; 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T11:53:09,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:09,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111526605cd41cd94f4ba5663be474d03c75_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:09,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742263_1439 (size=20074) 2024-11-15T11:53:09,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671589747, max timestamp=1731671589768Num uniques 859; 2024-11-15T11:53:09,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:09,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671589747, max timestamp=1731671589768Num uniques 859; 2024-11-15T11:53:09,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:09,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671589747, max timestamp=1731671589768Num uniques 859; 2024-11-15T11:53:09,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:09,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671649798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671649808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671649809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671649810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671649818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671649920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671649931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671649931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671649931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671649931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:09,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:10,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111526605cd41cd94f4ba5663be474d03c75_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526605cd41cd94f4ba5663be474d03c75_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:10,138 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b54cc7b6b7304e699e1b8068be629fad, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:10,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b54cc7b6b7304e699e1b8068be629fad is 175, key is test_row_0/A:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:10,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742264_1440 (size=57333) 2024-11-15T11:53:10,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671650141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671650141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671650141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671650150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671650150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671650461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:10,542 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b54cc7b6b7304e699e1b8068be629fad 2024-11-15T11:53:10,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/76c7770543e54e9089d9214ed8db66b1 is 50, key is test_row_0/B:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:10,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742265_1441 (size=12301) 2024-11-15T11:53:10,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/76c7770543e54e9089d9214ed8db66b1 2024-11-15T11:53:10,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3a38b8d5097043c9aa43a186430984b5 is 50, key is test_row_0/C:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:10,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742266_1442 (size=12301) 2024-11-15T11:53:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T11:53:10,691 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:10,691 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-15T11:53:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:10,693 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:10,694 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:10,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:10,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:10,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3a38b8d5097043c9aa43a186430984b5 2024-11-15T11:53:10,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/b54cc7b6b7304e699e1b8068be629fad as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad 2024-11-15T11:53:10,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad, entries=300, sequenceid=402, filesize=56.0 K 2024-11-15T11:53:10,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/76c7770543e54e9089d9214ed8db66b1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1 2024-11-15T11:53:10,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T11:53:10,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/3a38b8d5097043c9aa43a186430984b5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5 2024-11-15T11:53:10,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T11:53:10,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.50 KB/109056, currentSize=147.60 KB/151140 for 050c281702ae24e57ecff67ffc93194a in 1255ms, sequenceid=402, compaction requested=true 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:10,978 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:10,978 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 050c281702ae24e57ecff67ffc93194a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/A is initiating minor compaction (all files) 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/B is initiating minor compaction (all files) 2024-11-15T11:53:10,979 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/B in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,979 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/A in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,979 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/1b51d5a70ae14fbd93fb36b011db89f8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=36.6 K 2024-11-15T11:53:10,979 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=117.6 K 2024-11-15T11:53:10,979 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad] 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b51d5a70ae14fbd93fb36b011db89f8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96aafa77364b4c6394010a573cdf9ec3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 341df0572e2f4bf49f2242afe7e05c79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731671586288 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f5a59992ac486a9b400c54c152367a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731671586288 2024-11-15T11:53:10,979 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c7770543e54e9089d9214ed8db66b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671588558 2024-11-15T11:53:10,980 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b54cc7b6b7304e699e1b8068be629fad, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671588549 2024-11-15T11:53:10,984 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:10,985 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:10,985 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b6946f39304143e59ffe3405d85c6d5e is 50, key is test_row_0/B:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:10,988 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111538d8478f71f94b2e9f238f3a6e5f0cc1_050c281702ae24e57ecff67ffc93194a store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:10,990 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111538d8478f71f94b2e9f238f3a6e5f0cc1_050c281702ae24e57ecff67ffc93194a, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:10,990 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538d8478f71f94b2e9f238f3a6e5f0cc1_050c281702ae24e57ecff67ffc93194a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:10,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671589795, max timestamp=1731671590987Num uniques 1118; 2024-11-15T11:53:10,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-15T11:53:10,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731671589795, max timestamp=1731671590987Num uniques 1118; 2024-11-15T11:53:10,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-15T11:53:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:10,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=69.33 KB heapSize=188.95 KB 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:10,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:10,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742267_1443 (size=12949) 2024-11-15T11:53:10,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742268_1444 (size=4469) 2024-11-15T11:53:10,998 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#A#compaction#373 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:10,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:10,999 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/95139b9ae7b5473a8d6473946fc74f63 is 175, key is test_row_0/A:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:10,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,002 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/b6946f39304143e59ffe3405d85c6d5e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b6946f39304143e59ffe3405d85c6d5e 2024-11-15T11:53:11,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d64a44c1504545ec8a586e90bf2dc6a9_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671590992/Put/seqid=0 2024-11-15T11:53:11,006 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/B of 050c281702ae24e57ecff67ffc93194a into b6946f39304143e59ffe3405d85c6d5e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:11,006 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:11,006 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/B, priority=13, startTime=1731671590978; duration=0sec 2024-11-15T11:53:11,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742269_1445 (size=31903) 2024-11-15T11:53:11,006 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:11,006 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:B 2024-11-15T11:53:11,006 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:11,006 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:11,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 050c281702ae24e57ecff67ffc93194a/C is initiating minor compaction (all files) 2024-11-15T11:53:11,007 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 050c281702ae24e57ecff67ffc93194a/C in TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,007 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/e4e421af382f4b5185742150b6ae7be3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp, totalSize=36.6 K 2024-11-15T11:53:11,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e421af382f4b5185742150b6ae7be3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731671585907 2024-11-15T11:53:11,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a6e0f3e2723c4834a2ca9c71922fa972, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731671586288 2024-11-15T11:53:11,007 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a38b8d5097043c9aa43a186430984b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671588558 2024-11-15T11:53:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:11,014 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 050c281702ae24e57ecff67ffc93194a#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-15T11:53:11,015 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/40a0c67d9b8a4a188757acfd04b95cdf is 50, key is test_row_0/C:col10/1731671588558/Put/seqid=0 2024-11-15T11:53:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742270_1446 (size=17534) 2024-11-15T11:53:11,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742271_1447 (size=12949) 2024-11-15T11:53:11,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671651028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671651029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671651029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671651042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671651042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671651150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671651150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671651150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671651151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671651151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:11,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671651370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671651370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671651371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671651371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671651371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:11,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d64a44c1504545ec8a586e90bf2dc6a9_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d64a44c1504545ec8a586e90bf2dc6a9_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:11,424 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/95139b9ae7b5473a8d6473946fc74f63 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/95139b9ae7b5473a8d6473946fc74f63 2024-11-15T11:53:11,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/593fc5cfd2e54b15ad64f1c4104676c7, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:11,425 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/40a0c67d9b8a4a188757acfd04b95cdf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/40a0c67d9b8a4a188757acfd04b95cdf 2024-11-15T11:53:11,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/593fc5cfd2e54b15ad64f1c4104676c7 is 175, key is test_row_0/A:col10/1731671590992/Put/seqid=0 2024-11-15T11:53:11,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742272_1448 (size=48639) 2024-11-15T11:53:11,431 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/A of 050c281702ae24e57ecff67ffc93194a into 95139b9ae7b5473a8d6473946fc74f63(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:11,432 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/A, priority=13, startTime=1731671590978; duration=0sec 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:A 2024-11-15T11:53:11,432 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 050c281702ae24e57ecff67ffc93194a/C of 050c281702ae24e57ecff67ffc93194a into 40a0c67d9b8a4a188757acfd04b95cdf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:11,432 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., storeName=050c281702ae24e57ecff67ffc93194a/C, priority=13, startTime=1731671590978; duration=0sec 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:11,432 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 050c281702ae24e57ecff67ffc93194a:C 2024-11-15T11:53:11,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671651690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671651691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671651691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671651691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:11,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671651691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:11,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:11,761 INFO [Thread-1672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1672. 2024-11-15T11:53:11,761 DEBUG [Thread-1672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:11,761 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,761 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:11,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,768 INFO [Thread-1670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1670. 2024-11-15T11:53:11,768 DEBUG [Thread-1670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:11,768 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,768 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:11,769 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-11-15T11:53:11,769 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:11,769 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,769 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:11,772 INFO [Thread-1666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1666. 2024-11-15T11:53:11,772 DEBUG [Thread-1666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:11,772 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,772 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:11,778 INFO [Thread-1668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1668. 2024-11-15T11:53:11,778 DEBUG [Thread-1668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:11,778 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,778 DEBUG [Thread-1668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:11,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:11,832 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/593fc5cfd2e54b15ad64f1c4104676c7 2024-11-15T11:53:11,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/3d44b5b1d768421a9016ca3fe8554a30 is 50, key is test_row_0/B:col10/1731671590992/Put/seqid=0 2024-11-15T11:53:11,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742273_1449 (size=12301) 2024-11-15T11:53:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:11,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:12,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41810 deadline: 1731671652210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41846 deadline: 1731671652210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41834 deadline: 1731671652210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41800 deadline: 1731671652210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41844 deadline: 1731671652210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:12,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:12,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:12,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/3d44b5b1d768421a9016ca3fe8554a30 2024-11-15T11:53:12,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a692d50f9f1b450091408efa8bde1b4f is 50, key is test_row_0/C:col10/1731671590992/Put/seqid=0 2024-11-15T11:53:12,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742274_1450 (size=12301) 2024-11-15T11:53:12,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:12,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:12,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a692d50f9f1b450091408efa8bde1b4f 2024-11-15T11:53:12,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/593fc5cfd2e54b15ad64f1c4104676c7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/593fc5cfd2e54b15ad64f1c4104676c7 2024-11-15T11:53:12,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/593fc5cfd2e54b15ad64f1c4104676c7, entries=250, sequenceid=441, filesize=47.5 K 2024-11-15T11:53:12,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/3d44b5b1d768421a9016ca3fe8554a30 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/3d44b5b1d768421a9016ca3fe8554a30 2024-11-15T11:53:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/3d44b5b1d768421a9016ca3fe8554a30, entries=150, sequenceid=441, filesize=12.0 K 2024-11-15T11:53:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/a692d50f9f1b450091408efa8bde1b4f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a692d50f9f1b450091408efa8bde1b4f 2024-11-15T11:53:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a692d50f9f1b450091408efa8bde1b4f, entries=150, sequenceid=441, filesize=12.0 K 2024-11-15T11:53:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.04 KB/77860, heapSize ~208.83 KB/213840, currentSize=114.05 KB/116790 for 050c281702ae24e57ecff67ffc93194a in 1678ms, sequenceid=441, compaction requested=false 2024-11-15T11:53:12,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:12,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-15T11:53:12,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:12,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:12,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:12,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151e0c4954c2304cee8e0ac0dd992a3661_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671591011/Put/seqid=0 2024-11-15T11:53:12,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742275_1451 (size=12454) 2024-11-15T11:53:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:13,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:13,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151e0c4954c2304cee8e0ac0dd992a3661_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e0c4954c2304cee8e0ac0dd992a3661_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/d739a498409a421aa8bfb519437374bf, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:13,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/d739a498409a421aa8bfb519437374bf is 175, key is test_row_0/A:col10/1731671591011/Put/seqid=0 2024-11-15T11:53:13,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742276_1452 (size=31255) 2024-11-15T11:53:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:13,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. as already flushing 2024-11-15T11:53:13,224 INFO [Thread-1661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1661. 2024-11-15T11:53:13,224 DEBUG [Thread-1661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:13,224 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,224 DEBUG [Thread-1661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:13,225 INFO [Thread-1659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1659. 2024-11-15T11:53:13,225 DEBUG [Thread-1659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:13,225 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,225 INFO [Thread-1653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1653. 2024-11-15T11:53:13,225 DEBUG [Thread-1653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:13,225 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,225 DEBUG [Thread-1659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:13,225 DEBUG [Thread-1653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:13,226 INFO [Thread-1655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1655. 2024-11-15T11:53:13,226 DEBUG [Thread-1655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:13,226 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,226 DEBUG [Thread-1655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:13,232 INFO [Thread-1657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1657. 2024-11-15T11:53:13,232 DEBUG [Thread-1657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:13,232 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,232 DEBUG [Thread-1657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:13,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:13,510 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=464, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/d739a498409a421aa8bfb519437374bf 2024-11-15T11:53:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/03e6a693d7d745fd9e54fdde31735300 is 50, key is test_row_0/B:col10/1731671591011/Put/seqid=0 2024-11-15T11:53:13,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742277_1453 (size=12301) 2024-11-15T11:53:13,920 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/03e6a693d7d745fd9e54fdde31735300 2024-11-15T11:53:13,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/73b579e62f134c16bdfb217cb068387e is 50, key is test_row_0/C:col10/1731671591011/Put/seqid=0 2024-11-15T11:53:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742278_1454 (size=12301) 2024-11-15T11:53:14,212 DEBUG [regionserver/7adf9b3d9d04:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-15T11:53:14,233 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad] to archive 2024-11-15T11:53:14,235 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:14,239 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/e8f6311bcf0b46848fb3f372d131a496 2024-11-15T11:53:14,241 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b872eb0924e945b98320527d5c1231e3 2024-11-15T11:53:14,242 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/46426b33ce0b41fea725c2b4629ee5fd 2024-11-15T11:53:14,243 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/1e698a937d56468f8c818759e2b86d78 2024-11-15T11:53:14,244 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0bd3eb5b838f459eab93c77566b2b30c 2024-11-15T11:53:14,245 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/a5c6df296b734765b6baf2fba344f5ee 2024-11-15T11:53:14,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/af797ff8961d4195997d74cc0cc3b441 2024-11-15T11:53:14,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/99ff507bc6d3428a91b6df7a10e33071 2024-11-15T11:53:14,249 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/3b365bffa5ac40c2aa2dd634b03ff74e 2024-11-15T11:53:14,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/ca98db6bbaef401fa8024b872e90ac8e 2024-11-15T11:53:14,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/160889d1c2f142129750c2502ca10291 2024-11-15T11:53:14,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/11fe6a658aa24cbcaffa761046327667 2024-11-15T11:53:14,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/f7f657f4ef6e4711b555f9c33fb45201 2024-11-15T11:53:14,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/0c543a988c42404fb646509ce305a169 2024-11-15T11:53:14,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/fca90e5e7c0f41b19fe899e06e7b88bf 2024-11-15T11:53:14,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/221d81c20a544132ad13e78dcb565ced 2024-11-15T11:53:14,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/96aafa77364b4c6394010a573cdf9ec3 2024-11-15T11:53:14,260 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b2f5a59992ac486a9b400c54c152367a 2024-11-15T11:53:14,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/b54cc7b6b7304e699e1b8068be629fad 2024-11-15T11:53:14,264 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b599658e582049d5aafa1848b257f9f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f7916c6450b943e0a0120ae3dde0e5ce, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/26389b4eb2dd49dbb76ce142a87eec56, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f0735e5461cc4a63995da2e364264a02, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/1b51d5a70ae14fbd93fb36b011db89f8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1] to archive 2024-11-15T11:53:14,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:14,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c1092555ec374e098e6159c4a8459d34 2024-11-15T11:53:14,267 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/e3d9389903e44998bb6272929010978a 2024-11-15T11:53:14,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b599658e582049d5aafa1848b257f9f1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b599658e582049d5aafa1848b257f9f1 2024-11-15T11:53:14,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/aa153600246442f5879f63de0d9a5d48 2024-11-15T11:53:14,271 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c890a1738115483fbd8974c1df9dc97a 2024-11-15T11:53:14,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f7916c6450b943e0a0120ae3dde0e5ce to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f7916c6450b943e0a0120ae3dde0e5ce 2024-11-15T11:53:14,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b60e857b497942f2b96aeebd1bb12d60 2024-11-15T11:53:14,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9885744a5b534c12a6f492eb8b322427 2024-11-15T11:53:14,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/9341fbfb282147ffac121a599f79d68d 2024-11-15T11:53:14,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/26389b4eb2dd49dbb76ce142a87eec56 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/26389b4eb2dd49dbb76ce142a87eec56 2024-11-15T11:53:14,277 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/a96299d09f21487b8a207048c3d063c2 2024-11-15T11:53:14,279 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/57a8fefb9e034c58954812569ee1c97a 2024-11-15T11:53:14,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f0735e5461cc4a63995da2e364264a02 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/f0735e5461cc4a63995da2e364264a02 2024-11-15T11:53:14,281 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2a5ae11b89644367972cd32cb698463d 2024-11-15T11:53:14,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/c2ec75e4664e4eb1aecc910b84967bad 2024-11-15T11:53:14,283 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/1b51d5a70ae14fbd93fb36b011db89f8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/1b51d5a70ae14fbd93fb36b011db89f8 2024-11-15T11:53:14,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/00001b298f5642b5a8f8baf619245a3a 2024-11-15T11:53:14,285 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/341df0572e2f4bf49f2242afe7e05c79 2024-11-15T11:53:14,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/76c7770543e54e9089d9214ed8db66b1 2024-11-15T11:53:14,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73ff1e18b9ec416c912ee2b7135c7780, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/7e0537000fbc44efb766678a3649e83c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/417f524df2c649ce91eb65d74144f6a6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/c6e5f6f0939143c994426a46338dfe24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/e4e421af382f4b5185742150b6ae7be3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5] to archive 2024-11-15T11:53:14,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:14,293 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/ca4a8bf1be2444c39e2a6a541001742d 2024-11-15T11:53:14,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/8fa2830804ac455c9e2feb56f5ff1f4e 2024-11-15T11:53:14,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73ff1e18b9ec416c912ee2b7135c7780 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73ff1e18b9ec416c912ee2b7135c7780 2024-11-15T11:53:14,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3b6f8966ad51495998e84c337f293c8f 2024-11-15T11:53:14,297 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/267df7edc96d4b899e5900604cd163f7 2024-11-15T11:53:14,298 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/7e0537000fbc44efb766678a3649e83c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/7e0537000fbc44efb766678a3649e83c 2024-11-15T11:53:14,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4d26c967500648f8ac9fe5a72768116f 2024-11-15T11:53:14,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/cd0be3447f234351a34a3246422b3211 2024-11-15T11:53:14,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/49e64d7a50c74872a854586006bb5aa7 2024-11-15T11:53:14,303 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/417f524df2c649ce91eb65d74144f6a6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/417f524df2c649ce91eb65d74144f6a6 2024-11-15T11:53:14,304 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/87fedb39f81249eea18426b8f7ebb444 2024-11-15T11:53:14,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/be571e3ac8314c81b2e0bf95957a2f6e 2024-11-15T11:53:14,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/c6e5f6f0939143c994426a46338dfe24 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/c6e5f6f0939143c994426a46338dfe24 2024-11-15T11:53:14,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/4bd97d7d280e469bbfcb576dc91b74ae 2024-11-15T11:53:14,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/19e55959ff7e4fd1a86f16ee17b81191 2024-11-15T11:53:14,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/e4e421af382f4b5185742150b6ae7be3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/e4e421af382f4b5185742150b6ae7be3 2024-11-15T11:53:14,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/262ddb1a70be4d049a46d411a75bd284 2024-11-15T11:53:14,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a6e0f3e2723c4834a2ca9c71922fa972 2024-11-15T11:53:14,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7adf9b3d9d04:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/3a38b8d5097043c9aa43a186430984b5 2024-11-15T11:53:14,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/73b579e62f134c16bdfb217cb068387e 2024-11-15T11:53:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/d739a498409a421aa8bfb519437374bf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/d739a498409a421aa8bfb519437374bf 2024-11-15T11:53:14,345 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/d739a498409a421aa8bfb519437374bf, entries=150, sequenceid=464, filesize=30.5 K 2024-11-15T11:53:14,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/03e6a693d7d745fd9e54fdde31735300 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/03e6a693d7d745fd9e54fdde31735300 2024-11-15T11:53:14,348 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/03e6a693d7d745fd9e54fdde31735300, entries=150, sequenceid=464, filesize=12.0 K 2024-11-15T11:53:14,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/73b579e62f134c16bdfb217cb068387e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73b579e62f134c16bdfb217cb068387e 2024-11-15T11:53:14,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73b579e62f134c16bdfb217cb068387e, entries=150, sequenceid=464, filesize=12.0 K 2024-11-15T11:53:14,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 050c281702ae24e57ecff67ffc93194a in 1663ms, sequenceid=464, compaction requested=true 2024-11-15T11:53:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 050c281702ae24e57ecff67ffc93194a: 2024-11-15T11:53:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:14,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-15T11:53:14,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-15T11:53:14,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T11:53:14,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6590 sec 2024-11-15T11:53:14,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.6640 sec 2024-11-15T11:53:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T11:53:14,851 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:53:14,851 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3257 rows 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1098 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3266 rows 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1099 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3264 rows 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3237 rows 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-15T11:53:14,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3255 rows 2024-11-15T11:53:14,852 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:53:14,852 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:53:14,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:14,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:14,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:14,854 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:53:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:14,858 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671594858"}]},"ts":"1731671594858"} 2024-11-15T11:53:14,859 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:53:14,859 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:53:14,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:53:14,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, UNASSIGN}] 2024-11-15T11:53:14,863 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, UNASSIGN 2024-11-15T11:53:14,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:14,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, UNASSIGN because future has completed 2024-11-15T11:53:14,865 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:53:14,865 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:15,018 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 050c281702ae24e57ecff67ffc93194a, disabling compactions & flushes 2024-11-15T11:53:15,018 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. after waiting 0 ms 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:15,018 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 050c281702ae24e57ecff67ffc93194a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=A 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=B 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 050c281702ae24e57ecff67ffc93194a, store=C 2024-11-15T11:53:15,018 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:15,024 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2285530a3c846598c1d0dd85303eb60_050c281702ae24e57ecff67ffc93194a is 50, key is test_row_0/A:col10/1731671593224/Put/seqid=0 2024-11-15T11:53:15,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742279_1455 (size=12454) 2024-11-15T11:53:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:15,428 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:15,434 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2285530a3c846598c1d0dd85303eb60_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2285530a3c846598c1d0dd85303eb60_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:15,435 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/110a4e8d2ea3478f8174e7114c27c6a5, store: [table=TestAcidGuarantees family=A region=050c281702ae24e57ecff67ffc93194a] 2024-11-15T11:53:15,436 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/110a4e8d2ea3478f8174e7114c27c6a5 is 175, key is test_row_0/A:col10/1731671593224/Put/seqid=0 2024-11-15T11:53:15,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742280_1456 (size=31255) 2024-11-15T11:53:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:15,842 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/110a4e8d2ea3478f8174e7114c27c6a5 2024-11-15T11:53:15,848 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2f11e887a38c4c2fa62b6f129d847ee1 is 50, key is test_row_0/B:col10/1731671593224/Put/seqid=0 2024-11-15T11:53:15,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742281_1457 (size=12301) 2024-11-15T11:53:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:16,252 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2f11e887a38c4c2fa62b6f129d847ee1 2024-11-15T11:53:16,259 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/2d80a6619fa4494587e3af979ef5546d is 50, key is test_row_0/C:col10/1731671593224/Put/seqid=0 2024-11-15T11:53:16,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742282_1458 (size=12301) 2024-11-15T11:53:16,668 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/2d80a6619fa4494587e3af979ef5546d 2024-11-15T11:53:16,672 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/A/110a4e8d2ea3478f8174e7114c27c6a5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/110a4e8d2ea3478f8174e7114c27c6a5 2024-11-15T11:53:16,680 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/110a4e8d2ea3478f8174e7114c27c6a5, entries=150, sequenceid=472, filesize=30.5 K 2024-11-15T11:53:16,680 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/B/2f11e887a38c4c2fa62b6f129d847ee1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2f11e887a38c4c2fa62b6f129d847ee1 2024-11-15T11:53:16,683 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2f11e887a38c4c2fa62b6f129d847ee1, entries=150, sequenceid=472, filesize=12.0 K 2024-11-15T11:53:16,684 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/.tmp/C/2d80a6619fa4494587e3af979ef5546d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/2d80a6619fa4494587e3af979ef5546d 2024-11-15T11:53:16,686 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/2d80a6619fa4494587e3af979ef5546d, entries=150, sequenceid=472, filesize=12.0 K 2024-11-15T11:53:16,687 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 050c281702ae24e57ecff67ffc93194a in 1669ms, sequenceid=472, compaction requested=true 2024-11-15T11:53:16,691 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits/475.seqid, newMaxSeqId=475, maxSeqId=4 2024-11-15T11:53:16,691 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. 2024-11-15T11:53:16,691 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 050c281702ae24e57ecff67ffc93194a: Waiting for close lock at 1731671595018Running coprocessor pre-close hooks at 1731671595018Disabling compacts and flushes for region at 1731671595018Disabling writes for close at 1731671595018Obtaining lock to block concurrent updates at 1731671595018Preparing flush snapshotting stores in 050c281702ae24e57ecff67ffc93194a at 1731671595018Finished memstore snapshotting TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731671595018Flushing stores of TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a. at 1731671595019 (+1 ms)Flushing 050c281702ae24e57ecff67ffc93194a/A: creating writer at 1731671595019Flushing mob file 050c281702ae24e57ecff67ffc93194a/A: appending metadata at 1731671595024 (+5 ms)Flushing mob file 050c281702ae24e57ecff67ffc93194a/A: closing flushed file at 1731671595024Flushing 050c281702ae24e57ecff67ffc93194a/A: appending metadata at 1731671595435 (+411 ms)Flushing 050c281702ae24e57ecff67ffc93194a/A: closing flushed file at 1731671595435Flushing 050c281702ae24e57ecff67ffc93194a/B: creating writer at 1731671595846 (+411 ms)Flushing 050c281702ae24e57ecff67ffc93194a/B: appending metadata at 1731671595848 (+2 ms)Flushing 050c281702ae24e57ecff67ffc93194a/B: closing flushed file at 1731671595848Flushing 050c281702ae24e57ecff67ffc93194a/C: creating writer at 1731671596256 (+408 ms)Flushing 050c281702ae24e57ecff67ffc93194a/C: appending metadata at 1731671596258 (+2 ms)Flushing 050c281702ae24e57ecff67ffc93194a/C: closing flushed file at 1731671596258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595716b: reopening flushed file at 1731671596672 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24be4b95: reopening flushed file at 1731671596680 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c886ce4: reopening flushed file at 1731671596683 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 050c281702ae24e57ecff67ffc93194a in 1669ms, sequenceid=472, compaction requested=true at 1731671596687 (+4 ms)Writing region close event to WAL at 1731671596688 (+1 ms)Running coprocessor post-close hooks at 1731671596691 (+3 ms)Closed at 1731671596691 2024-11-15T11:53:16,693 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:16,693 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=050c281702ae24e57ecff67ffc93194a, regionState=CLOSED 2024-11-15T11:53:16,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:16,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T11:53:16,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 050c281702ae24e57ecff67ffc93194a, server=7adf9b3d9d04,44479,1731671471201 in 1.8300 sec 2024-11-15T11:53:16,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-15T11:53:16,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=050c281702ae24e57ecff67ffc93194a, UNASSIGN in 1.8340 sec 2024-11-15T11:53:16,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T11:53:16,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8380 sec 2024-11-15T11:53:16,700 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671596700"}]},"ts":"1731671596700"} 2024-11-15T11:53:16,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:53:16,701 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:53:16,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8470 sec 2024-11-15T11:53:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T11:53:17,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:17,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:53:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,004 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,004 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T11:53:17,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,008 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits] 2024-11-15T11:53:17,010 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/110a4e8d2ea3478f8174e7114c27c6a5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/110a4e8d2ea3478f8174e7114c27c6a5 2024-11-15T11:53:17,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/593fc5cfd2e54b15ad64f1c4104676c7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/593fc5cfd2e54b15ad64f1c4104676c7 2024-11-15T11:53:17,011 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/95139b9ae7b5473a8d6473946fc74f63 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/95139b9ae7b5473a8d6473946fc74f63 2024-11-15T11:53:17,012 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/d739a498409a421aa8bfb519437374bf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/A/d739a498409a421aa8bfb519437374bf 2024-11-15T11:53:17,014 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/03e6a693d7d745fd9e54fdde31735300 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/03e6a693d7d745fd9e54fdde31735300 2024-11-15T11:53:17,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2f11e887a38c4c2fa62b6f129d847ee1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/2f11e887a38c4c2fa62b6f129d847ee1 2024-11-15T11:53:17,016 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/3d44b5b1d768421a9016ca3fe8554a30 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/3d44b5b1d768421a9016ca3fe8554a30 2024-11-15T11:53:17,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b6946f39304143e59ffe3405d85c6d5e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/B/b6946f39304143e59ffe3405d85c6d5e 2024-11-15T11:53:17,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/2d80a6619fa4494587e3af979ef5546d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/2d80a6619fa4494587e3af979ef5546d 2024-11-15T11:53:17,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/40a0c67d9b8a4a188757acfd04b95cdf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/40a0c67d9b8a4a188757acfd04b95cdf 2024-11-15T11:53:17,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73b579e62f134c16bdfb217cb068387e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/73b579e62f134c16bdfb217cb068387e 2024-11-15T11:53:17,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a692d50f9f1b450091408efa8bde1b4f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/C/a692d50f9f1b450091408efa8bde1b4f 2024-11-15T11:53:17,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits/475.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a/recovered.edits/475.seqid 2024-11-15T11:53:17,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,023 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:53:17,024 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:53:17,024 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T11:53:17,026 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e0c4954c2304cee8e0ac0dd992a3661_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151e0c4954c2304cee8e0ac0dd992a3661_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,027 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151fae362ea72d45c3a30480bdc7b0d2ad_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151fae362ea72d45c3a30480bdc7b0d2ad_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,028 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152074042e57ff4fa0bda19fdab7df771a_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152074042e57ff4fa0bda19fdab7df771a_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,029 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522029777e3214fc0a454e02d0e88e1cb_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111522029777e3214fc0a454e02d0e88e1cb_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,030 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526605cd41cd94f4ba5663be474d03c75_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111526605cd41cd94f4ba5663be474d03c75_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,031 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111535ff65fe2ecb4c37af4b4a62a1feb385_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111535ff65fe2ecb4c37af4b4a62a1feb385_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,032 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154f6c0648301343468b9821c580e365e9_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154f6c0648301343468b9821c580e365e9_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,033 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115537d7d1a528a43e3ab6846615168842d_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115537d7d1a528a43e3ab6846615168842d_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,034 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e597dd706304fba8dda800c9f5e275e_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e597dd706304fba8dda800c9f5e275e_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,035 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ee3d62af12c4a25aeeabca7f530d4c0_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ee3d62af12c4a25aeeabca7f530d4c0_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,036 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a13b1b60324f8191442d65c20a08e8_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a13b1b60324f8191442d65c20a08e8_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,037 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115880f38a6b8294cc0a8c1e501ca7cacb4_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115880f38a6b8294cc0a8c1e501ca7cacb4_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,038 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596a63b4e18754a54b2b321796fb96789_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111596a63b4e18754a54b2b321796fb96789_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,039 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2285530a3c846598c1d0dd85303eb60_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b2285530a3c846598c1d0dd85303eb60_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,040 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba6d212295d44e54ab8f2ff635b22427_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ba6d212295d44e54ab8f2ff635b22427_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,041 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d64a44c1504545ec8a586e90bf2dc6a9_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d64a44c1504545ec8a586e90bf2dc6a9_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,041 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dda9d2e31c59499180bd4f7bdef48a24_050c281702ae24e57ecff67ffc93194a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dda9d2e31c59499180bd4f7bdef48a24_050c281702ae24e57ecff67ffc93194a 2024-11-15T11:53:17,042 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:53:17,043 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,045 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:53:17,047 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:53:17,048 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,048 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:53:17,048 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671597048"}]},"ts":"9223372036854775807"} 2024-11-15T11:53:17,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:53:17,050 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 050c281702ae24e57ecff67ffc93194a, NAME => 'TestAcidGuarantees,,1731671568535.050c281702ae24e57ecff67ffc93194a.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:53:17,050 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:53:17,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671597050"}]},"ts":"9223372036854775807"} 2024-11-15T11:53:17,052 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:53:17,052 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-15T11:53:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T11:53:17,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:53:17,113 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:17,122 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=529 (was 485) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11631 (was 10209) - AvailableMemoryMB LEAK? - 2024-11-15T11:53:17,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=529, ProcessCount=11, AvailableMemoryMB=11629 2024-11-15T11:53:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:53:17,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:53:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:17,136 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:53:17,136 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:17,136 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-15T11:53:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:17,137 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:53:17,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742283_1459 (size=1061) 2024-11-15T11:53:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:17,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:53:17,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742284_1460 (size=53) 2024-11-15T11:53:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f47e749e6316d0ed90d8985c2e855dcb, disabling compactions & flushes 2024-11-15T11:53:17,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. after waiting 0 ms 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:17,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:17,954 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f47e749e6316d0ed90d8985c2e855dcb: Waiting for close lock at 1731671597954Disabling compacts and flushes for region at 1731671597954Disabling writes for close at 1731671597954Writing region close event to WAL at 1731671597954Closed at 1731671597954 2024-11-15T11:53:17,956 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:53:17,956 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671597956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671597956"}]},"ts":"1731671597956"} 2024-11-15T11:53:17,958 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:53:17,958 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:53:17,959 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671597959"}]},"ts":"1731671597959"} 2024-11-15T11:53:17,960 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:53:17,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, ASSIGN}] 2024-11-15T11:53:17,962 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, ASSIGN 2024-11-15T11:53:17,963 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:53:18,114 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f47e749e6316d0ed90d8985c2e855dcb, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:18,116 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, ASSIGN because future has completed 2024-11-15T11:53:18,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:18,270 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:18,270 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:53:18,271 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:18,271 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:53:18,271 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,271 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,272 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,273 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:18,273 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47e749e6316d0ed90d8985c2e855dcb columnFamilyName A 2024-11-15T11:53:18,273 DEBUG [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:18,273 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(327): Store=f47e749e6316d0ed90d8985c2e855dcb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:18,273 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,274 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:18,274 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47e749e6316d0ed90d8985c2e855dcb columnFamilyName B 2024-11-15T11:53:18,274 DEBUG [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:18,275 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(327): Store=f47e749e6316d0ed90d8985c2e855dcb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:18,275 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,275 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:18,276 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f47e749e6316d0ed90d8985c2e855dcb columnFamilyName C 2024-11-15T11:53:18,276 DEBUG [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:18,276 INFO [StoreOpener-f47e749e6316d0ed90d8985c2e855dcb-1 {}] regionserver.HStore(327): Store=f47e749e6316d0ed90d8985c2e855dcb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:18,276 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:18,276 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,277 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,277 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,278 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,278 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,278 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:53:18,279 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,281 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:53:18,281 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened f47e749e6316d0ed90d8985c2e855dcb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74720642, jitterRate=0.11342433094978333}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:53:18,281 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:18,282 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for f47e749e6316d0ed90d8985c2e855dcb: Running coprocessor pre-open hook at 1731671598271Writing region info on filesystem at 1731671598271Initializing all the Stores at 1731671598272 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671598272Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671598272Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671598272Cleaning up temporary data from old regions at 1731671598278 (+6 ms)Running coprocessor post-open hooks at 1731671598281 (+3 ms)Region opened successfully at 1731671598282 (+1 ms) 2024-11-15T11:53:18,282 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., pid=122, masterSystemTime=1731671598267 2024-11-15T11:53:18,284 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:18,284 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:18,284 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=f47e749e6316d0ed90d8985c2e855dcb, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:18,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:18,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T11:53:18,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 in 171 msec 2024-11-15T11:53:18,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-15T11:53:18,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, ASSIGN in 328 msec 2024-11-15T11:53:18,291 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:53:18,291 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671598291"}]},"ts":"1731671598291"} 2024-11-15T11:53:18,292 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:53:18,293 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:53:18,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-15T11:53:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-15T11:53:19,291 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:19,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d9c874d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212347ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,298 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:53:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:53:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4df2c421 2024-11-15T11:53:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:53:19,299 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:53:19,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,301 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515f40fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,303 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d592f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,305 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f634806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f876c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,308 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae192fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd2083b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7a9d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e837165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,315 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7671d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8ee90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecd081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,320 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29757b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,324 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,325 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1cff17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20822bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,328 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e98925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b63d761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,332 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255cf9c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:19,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:19,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37893894, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:19,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9424d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:19,340 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,341 DEBUG [Thread-2068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,341 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,345 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,346 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,347 DEBUG [Thread-2061 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,347 DEBUG [Thread-2061 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,351 DEBUG [Thread-2072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,353 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,353 DEBUG [Thread-2066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,354 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,355 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:19,355 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T11:53:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:19,359 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:19,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:53:19,360 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:19,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:19,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:19,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:19,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:19,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:19,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,367 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:19,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2] 2024-11-15T11:53:19,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/725ee2717341452e9183109a8cd23d66 is 50, key is test_row_0/A:col10/1731671599356/Put/seqid=0 2024-11-15T11:53:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742285_1461 (size=14341) 2024-11-15T11:53:19,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/725ee2717341452e9183109a8cd23d66 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671599391, max timestamp=1731671599410Num uniques 859; 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671599391, max timestamp=1731671599410Num uniques 859; 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671599391, max timestamp=1731671599410Num uniques 859; 2024-11-15T11:53:19,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:19,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/2513df9e059445f7bf28b1d0c5aefb81 is 50, key is test_row_0/B:col10/1731671599356/Put/seqid=0 2024-11-15T11:53:19,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742286_1462 (size=12001) 2024-11-15T11:53:19,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671659452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671659453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:19,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671659456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671659457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671659457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:19,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:19,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671659560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671659570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671659580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671659581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671659581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,592 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:19,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671659770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671659780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671659801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671659801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671659801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:19,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:19,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:19,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/2513df9e059445f7bf28b1d0c5aefb81 2024-11-15T11:53:19,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/50562059f9674be29bddc77804cc9e1c is 50, key is test_row_0/C:col10/1731671599356/Put/seqid=0 2024-11-15T11:53:19,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742287_1463 (size=12001) 2024-11-15T11:53:19,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:19,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:19,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:20,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671660090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671660090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671660121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671660121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671660123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/50562059f9674be29bddc77804cc9e1c 2024-11-15T11:53:20,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:20,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/725ee2717341452e9183109a8cd23d66 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66 2024-11-15T11:53:20,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66, entries=200, sequenceid=13, filesize=14.0 K 2024-11-15T11:53:20,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/2513df9e059445f7bf28b1d0c5aefb81 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81 2024-11-15T11:53:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T11:53:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/50562059f9674be29bddc77804cc9e1c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c 2024-11-15T11:53:20,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T11:53:20,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f47e749e6316d0ed90d8985c2e855dcb in 933ms, sequenceid=13, compaction requested=false 2024-11-15T11:53:20,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:20,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T11:53:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,431 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:20,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8027a02842ec4a6c8798a3072651c163 is 50, key is test_row_0/A:col10/1731671599435/Put/seqid=0 2024-11-15T11:53:20,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742288_1464 (size=12001) 2024-11-15T11:53:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:20,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:20,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671660643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671660656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671660657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671660659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671660662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671660771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671660771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671660780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671660791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671660791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:20,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:20,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8027a02842ec4a6c8798a3072651c163 2024-11-15T11:53:20,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/adf8f154dbde4e07a2c6d403ad3047f4 is 50, key is test_row_0/B:col10/1731671599435/Put/seqid=0 2024-11-15T11:53:20,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742289_1465 (size=12001) 2024-11-15T11:53:20,880 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/adf8f154dbde4e07a2c6d403ad3047f4 2024-11-15T11:53:20,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a7854062a147444e80f7fcae3819f578 is 50, key is test_row_0/C:col10/1731671599435/Put/seqid=0 2024-11-15T11:53:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742290_1466 (size=12001) 2024-11-15T11:53:20,906 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a7854062a147444e80f7fcae3819f578 2024-11-15T11:53:20,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8027a02842ec4a6c8798a3072651c163 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163 2024-11-15T11:53:20,921 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163, entries=150, sequenceid=47, filesize=11.7 K 2024-11-15T11:53:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/adf8f154dbde4e07a2c6d403ad3047f4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4 2024-11-15T11:53:20,927 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-15T11:53:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a7854062a147444e80f7fcae3819f578 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578 2024-11-15T11:53:20,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578, entries=150, sequenceid=47, filesize=11.7 K 2024-11-15T11:53:20,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for f47e749e6316d0ed90d8985c2e855dcb in 503ms, sequenceid=47, compaction requested=false 2024-11-15T11:53:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T11:53:20,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T11:53:20,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T11:53:20,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5780 sec 2024-11-15T11:53:20,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5850 sec 2024-11-15T11:53:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:20,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:20,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/04e801348c124712b56b64917a7ee8d6 is 50, key is test_row_0/A:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742291_1467 (size=14341) 2024-11-15T11:53:21,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/04e801348c124712b56b64917a7ee8d6 2024-11-15T11:53:21,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/05910922ce194612b1614ce56dbbbfe0 is 50, key is test_row_0/B:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742292_1468 (size=12001) 2024-11-15T11:53:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671661024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671661025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671661027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671661038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671661038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671661140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671661141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671661141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671661150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671661160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671661351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671661351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671661351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671661360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671661382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/05910922ce194612b1614ce56dbbbfe0 2024-11-15T11:53:21,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 is 50, key is test_row_0/C:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742293_1469 (size=12001) 2024-11-15T11:53:21,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 2024-11-15T11:53:21,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/04e801348c124712b56b64917a7ee8d6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6 2024-11-15T11:53:21,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6, entries=200, sequenceid=60, filesize=14.0 K 2024-11-15T11:53:21,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/05910922ce194612b1614ce56dbbbfe0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0 2024-11-15T11:53:21,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T11:53:21,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 2024-11-15T11:53:21,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198, entries=150, sequenceid=60, filesize=11.7 K 2024-11-15T11:53:21,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f47e749e6316d0ed90d8985c2e855dcb in 474ms, sequenceid=60, compaction requested=true 2024-11-15T11:53:21,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:21,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:21,468 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:21,468 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,468 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,469 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=39.7 K 2024-11-15T11:53:21,469 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=35.2 K 2024-11-15T11:53:21,469 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2513df9e059445f7bf28b1d0c5aefb81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671599351 2024-11-15T11:53:21,469 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725ee2717341452e9183109a8cd23d66, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671599351 2024-11-15T11:53:21,469 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting adf8f154dbde4e07a2c6d403ad3047f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731671599435 2024-11-15T11:53:21,469 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8027a02842ec4a6c8798a3072651c163, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731671599435 2024-11-15T11:53:21,470 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 05910922ce194612b1614ce56dbbbfe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:21,470 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e801348c124712b56b64917a7ee8d6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:21,481 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:21,481 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:21,481 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ed9e7af323e048929e8b6568e77c8894 is 50, key is test_row_0/B:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,481 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/4110ac530ab44c498a1d226e87e41bae is 50, key is test_row_0/A:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742295_1471 (size=12104) 2024-11-15T11:53:21,509 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/4110ac530ab44c498a1d226e87e41bae as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/4110ac530ab44c498a1d226e87e41bae 2024-11-15T11:53:21,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742294_1470 (size=12104) 2024-11-15T11:53:21,516 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ed9e7af323e048929e8b6568e77c8894 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ed9e7af323e048929e8b6568e77c8894 2024-11-15T11:53:21,517 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into 4110ac530ab44c498a1d226e87e41bae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:21,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:21,517 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=13, startTime=1731671601467; duration=0sec 2024-11-15T11:53:21,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:21,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:21,517 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:21,518 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:21,518 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:21,518 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,518 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=35.2 K 2024-11-15T11:53:21,518 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50562059f9674be29bddc77804cc9e1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731671599351 2024-11-15T11:53:21,519 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7854062a147444e80f7fcae3819f578, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731671599435 2024-11-15T11:53:21,521 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into ed9e7af323e048929e8b6568e77c8894(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:21,521 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ada7cbb3d4d4bf4999c2a8e71ab1198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:21,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:21,521 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=13, startTime=1731671601468; duration=0sec 2024-11-15T11:53:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T11:53:21,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:21,521 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:21,521 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:21,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T11:53:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:21,524 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:21,524 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:21,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:21,531 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:21,531 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/49b4f2bef75f47f7983bf10f85edfb2d is 50, key is test_row_0/C:col10/1731671600992/Put/seqid=0 2024-11-15T11:53:21,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742296_1472 (size=12104) 2024-11-15T11:53:21,556 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/49b4f2bef75f47f7983bf10f85edfb2d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/49b4f2bef75f47f7983bf10f85edfb2d 2024-11-15T11:53:21,569 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into 49b4f2bef75f47f7983bf10f85edfb2d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:21,569 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:21,569 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=13, startTime=1731671601468; duration=0sec 2024-11-15T11:53:21,569 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:21,569 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:21,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=114.05 KB heapSize=339.41 KB 2024-11-15T11:53:21,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671601025, max timestamp=1731671601665Num uniques 859; 2024-11-15T11:53:21,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:21,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671601025, max timestamp=1731671601665Num uniques 859; 2024-11-15T11:53:21,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:21,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671601025, max timestamp=1731671601665Num uniques 859; 2024-11-15T11:53:21,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:21,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:21,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6163edbb0b254fd49720a138a7371ede is 50, key is test_row_0/A:col10/1731671601025/Put/seqid=0 2024-11-15T11:53:21,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742297_1473 (size=14341) 2024-11-15T11:53:21,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6163edbb0b254fd49720a138a7371ede 2024-11-15T11:53:21,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/809ac8bb2fff4523b6bd1e08f5b3dd1f is 50, key is test_row_0/B:col10/1731671601025/Put/seqid=0 2024-11-15T11:53:21,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671601716, max timestamp=1731671601739Num uniques 859; 2024-11-15T11:53:21,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671601716, max timestamp=1731671601739Num uniques 859; 2024-11-15T11:53:21,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:21,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:21,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671601716, max timestamp=1731671601739Num uniques 859; 2024-11-15T11:53:21,743 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742298_1474 (size=12001) 2024-11-15T11:53:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731671601775, max timestamp=1731671601802Num uniques 1068; 2024-11-15T11:53:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-15T11:53:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731671601775, max timestamp=1731671601802Num uniques 1068; 2024-11-15T11:53:21,806 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-15T11:53:21,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1731671601775, max timestamp=1731671601802Num uniques 1068; 2024-11-15T11:53:21,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=16848, suffix heap size=139040,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=368 2024-11-15T11:53:21,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:21,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671661844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671661845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671661852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671661854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671661856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671661971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671661971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671661971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671661971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:21,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:21,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:21,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671661980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:22,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/809ac8bb2fff4523b6bd1e08f5b3dd1f 2024-11-15T11:53:22,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/0950a0efbe4a4843af3036f75ceb23b2 is 50, key is test_row_0/C:col10/1731671601025/Put/seqid=0 2024-11-15T11:53:22,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742299_1475 (size=12001) 2024-11-15T11:53:22,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671662180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671662180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671662190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671662190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671662201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,232 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:53:22,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:22,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:22,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671662501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671662501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671662501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671662501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671662520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:22,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/0950a0efbe4a4843af3036f75ceb23b2 2024-11-15T11:53:22,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6163edbb0b254fd49720a138a7371ede as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede 2024-11-15T11:53:22,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede, entries=200, sequenceid=84, filesize=14.0 K 2024-11-15T11:53:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/809ac8bb2fff4523b6bd1e08f5b3dd1f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f 2024-11-15T11:53:22,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T11:53:22,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/0950a0efbe4a4843af3036f75ceb23b2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2 2024-11-15T11:53:22,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T11:53:22,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.31 KB/32060, heapSize ~76.34 KB/78176, currentSize=161.02 KB/164880 for f47e749e6316d0ed90d8985c2e855dcb in 927ms, sequenceid=84, compaction requested=false 2024-11-15T11:53:22,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:22,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T11:53:22,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:22,597 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-11-15T11:53:22,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:22,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/23f0971f40be4771bf48ff2deeebae6c is 50, key is test_row_0/A:col10/1731671601853/Put/seqid=0 2024-11-15T11:53:22,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742300_1476 (size=12151) 2024-11-15T11:53:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:23,014 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/23f0971f40be4771bf48ff2deeebae6c 2024-11-15T11:53:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:23,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671663016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671663017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671663019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fb03a90dda7840b8a7a3312e170e666b is 50, key is test_row_0/B:col10/1731671601853/Put/seqid=0 2024-11-15T11:53:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671663020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742301_1477 (size=12151) 2024-11-15T11:53:23,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671663040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671663130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671663130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671663130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671663140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671663351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671663351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671663351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671663352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,436 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fb03a90dda7840b8a7a3312e170e666b 2024-11-15T11:53:23,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/7ec5ba911fdd4f47bc729c7aaa76933b is 50, key is test_row_0/C:col10/1731671601853/Put/seqid=0 2024-11-15T11:53:23,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742302_1478 (size=12151) 2024-11-15T11:53:23,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671663671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:23,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:23,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/7ec5ba911fdd4f47bc729c7aaa76933b 2024-11-15T11:53:23,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/23f0971f40be4771bf48ff2deeebae6c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c 2024-11-15T11:53:23,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c, entries=150, sequenceid=138, filesize=11.9 K 2024-11-15T11:53:23,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fb03a90dda7840b8a7a3312e170e666b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b 2024-11-15T11:53:23,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b, entries=150, sequenceid=138, filesize=11.9 K 2024-11-15T11:53:23,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/7ec5ba911fdd4f47bc729c7aaa76933b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b 2024-11-15T11:53:23,860 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b, entries=150, sequenceid=138, filesize=11.9 K 2024-11-15T11:53:23,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=26.84 KB/27480 for f47e749e6316d0ed90d8985c2e855dcb in 1264ms, sequenceid=138, compaction requested=true 2024-11-15T11:53:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:23,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:23,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T11:53:23,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T11:53:23,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T11:53:23,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-15T11:53:23,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3440 sec 2024-11-15T11:53:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:24,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:24,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:24,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:24,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/ac1aadc35c3a4dc0b5584ed6caa488ba is 50, key is test_row_0/A:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742303_1479 (size=14541) 2024-11-15T11:53:24,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/ac1aadc35c3a4dc0b5584ed6caa488ba 2024-11-15T11:53:24,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a66b05fd1d6947d9abb19cd611d29a73 is 50, key is test_row_0/B:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742304_1480 (size=12151) 2024-11-15T11:53:24,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a66b05fd1d6947d9abb19cd611d29a73 2024-11-15T11:53:24,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/efc925cbf3df48acb7c19c22126bf837 is 50, key is test_row_0/C:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742305_1481 (size=12151) 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731671604087, max timestamp=1731671604204Num uniques 859; 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731671604087, max timestamp=1731671604204Num uniques 859; 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731671604087, max timestamp=1731671604204Num uniques 859; 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T11:53:24,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-15T11:53:24,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671664236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671664237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671664238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671664239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671664243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671664351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671664361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671664361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671664361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671664361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/efc925cbf3df48acb7c19c22126bf837 2024-11-15T11:53:24,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/ac1aadc35c3a4dc0b5584ed6caa488ba as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba 2024-11-15T11:53:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba, entries=200, sequenceid=148, filesize=14.2 K 2024-11-15T11:53:24,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a66b05fd1d6947d9abb19cd611d29a73 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73 2024-11-15T11:53:24,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73, entries=150, sequenceid=148, filesize=11.9 K 2024-11-15T11:53:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/efc925cbf3df48acb7c19c22126bf837 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837 2024-11-15T11:53:24,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837, entries=150, sequenceid=148, filesize=11.9 K 2024-11-15T11:53:24,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f47e749e6316d0ed90d8985c2e855dcb in 472ms, sequenceid=148, compaction requested=true 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:24,545 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:24,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:24,545 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:24,546 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53137 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:24,546 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:24,546 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:24,546 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:24,546 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:24,546 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:24,547 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/4110ac530ab44c498a1d226e87e41bae, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=51.9 K 2024-11-15T11:53:24,547 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ed9e7af323e048929e8b6568e77c8894, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=47.3 K 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4110ac530ab44c498a1d226e87e41bae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ed9e7af323e048929e8b6568e77c8894, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6163edbb0b254fd49720a138a7371ede, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671601025 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 809ac8bb2fff4523b6bd1e08f5b3dd1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671601025 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f0971f40be4771bf48ff2deeebae6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731671601832 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fb03a90dda7840b8a7a3312e170e666b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731671601832 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac1aadc35c3a4dc0b5584ed6caa488ba, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603017 2024-11-15T11:53:24,547 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a66b05fd1d6947d9abb19cd611d29a73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603018 2024-11-15T11:53:24,554 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:24,554 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:24,555 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/ae0cc90cbeac429aaaf98274d9570db1 is 50, key is test_row_0/A:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,555 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/0a61a548d1f34106a01d5cdc0cdfc942 is 50, key is test_row_0/B:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742306_1482 (size=12391) 2024-11-15T11:53:24,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742307_1483 (size=12391) 2024-11-15T11:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:24,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:24,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:24,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/3ce1b5a013594a34b126e9d819583cff is 50, key is test_row_0/A:col10/1731671604241/Put/seqid=0 2024-11-15T11:53:24,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742308_1484 (size=14541) 2024-11-15T11:53:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671664581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671664585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671664588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671664589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671664590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671664700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671664711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671664711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671664920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671664920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671664920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671664930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671664931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:24,963 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/0a61a548d1f34106a01d5cdc0cdfc942 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/0a61a548d1f34106a01d5cdc0cdfc942 2024-11-15T11:53:24,963 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/ae0cc90cbeac429aaaf98274d9570db1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ae0cc90cbeac429aaaf98274d9570db1 2024-11-15T11:53:24,968 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into ae0cc90cbeac429aaaf98274d9570db1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:24,968 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into 0a61a548d1f34106a01d5cdc0cdfc942(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:24,968 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=12, startTime=1731671604545; duration=0sec 2024-11-15T11:53:24,968 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=12, startTime=1731671604545; duration=0sec 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:24,968 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:24,969 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:24,969 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:24,969 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:24,969 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/49b4f2bef75f47f7983bf10f85edfb2d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=47.3 K 2024-11-15T11:53:24,970 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b4f2bef75f47f7983bf10f85edfb2d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731671600623 2024-11-15T11:53:24,970 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0950a0efbe4a4843af3036f75ceb23b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671601025 2024-11-15T11:53:24,970 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec5ba911fdd4f47bc729c7aaa76933b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731671601832 2024-11-15T11:53:24,970 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc925cbf3df48acb7c19c22126bf837, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603018 2024-11-15T11:53:24,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/3ce1b5a013594a34b126e9d819583cff 2024-11-15T11:53:24,981 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:24,981 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ef30ba4db6f44afb8f7f81d5b4b7b931 is 50, key is test_row_0/C:col10/1731671604072/Put/seqid=0 2024-11-15T11:53:24,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a0e0cdb6844644f493a7085709575fb1 is 50, key is test_row_0/B:col10/1731671604241/Put/seqid=0 2024-11-15T11:53:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742310_1486 (size=12151) 2024-11-15T11:53:24,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742309_1485 (size=12391) 2024-11-15T11:53:25,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671665230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671665230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671665230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671665251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671665251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a0e0cdb6844644f493a7085709575fb1 2024-11-15T11:53:25,396 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ef30ba4db6f44afb8f7f81d5b4b7b931 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef30ba4db6f44afb8f7f81d5b4b7b931 2024-11-15T11:53:25,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/31a0621930e243b98ad365d052f540a4 is 50, key is test_row_0/C:col10/1731671604241/Put/seqid=0 2024-11-15T11:53:25,400 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into ef30ba4db6f44afb8f7f81d5b4b7b931(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:25,400 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:25,400 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=12, startTime=1731671604545; duration=0sec 2024-11-15T11:53:25,400 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:25,400 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:25,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742311_1487 (size=12151) 2024-11-15T11:53:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T11:53:25,701 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:25,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T11:53:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T11:53:25,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:25,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:25,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671665750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671665750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671665750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671665761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671665761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:25,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:25,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/31a0621930e243b98ad365d052f540a4 2024-11-15T11:53:25,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/3ce1b5a013594a34b126e9d819583cff as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff 2024-11-15T11:53:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T11:53:25,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff, entries=200, sequenceid=183, filesize=14.2 K 2024-11-15T11:53:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/a0e0cdb6844644f493a7085709575fb1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1 2024-11-15T11:53:25,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1, entries=150, sequenceid=183, filesize=11.9 K 2024-11-15T11:53:25,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/31a0621930e243b98ad365d052f540a4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4 2024-11-15T11:53:25,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4, entries=150, sequenceid=183, filesize=11.9 K 2024-11-15T11:53:25,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for f47e749e6316d0ed90d8985c2e855dcb in 1258ms, sequenceid=183, compaction requested=false 2024-11-15T11:53:25,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:25,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T11:53:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:25,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/aabfa3075aff40ae8fd3a3c0a2b2af4f is 50, key is test_row_0/A:col10/1731671604589/Put/seqid=0 2024-11-15T11:53:25,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742312_1488 (size=12151) 2024-11-15T11:53:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T11:53:26,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/aabfa3075aff40ae8fd3a3c0a2b2af4f 2024-11-15T11:53:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/b0278442bdb047bd87d5af09d62dc1a6 is 50, key is test_row_0/B:col10/1731671604589/Put/seqid=0 2024-11-15T11:53:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742313_1489 (size=12151) 2024-11-15T11:53:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T11:53:26,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/b0278442bdb047bd87d5af09d62dc1a6 2024-11-15T11:53:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f98c96b0d3a0484c891fc770f7449f7b is 50, key is test_row_0/C:col10/1731671604589/Put/seqid=0 2024-11-15T11:53:26,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742314_1490 (size=12151) 2024-11-15T11:53:26,700 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f98c96b0d3a0484c891fc770f7449f7b 2024-11-15T11:53:26,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/aabfa3075aff40ae8fd3a3c0a2b2af4f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f 2024-11-15T11:53:26,716 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:53:26,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/b0278442bdb047bd87d5af09d62dc1a6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6 2024-11-15T11:53:26,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:53:26,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f98c96b0d3a0484c891fc770f7449f7b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b 2024-11-15T11:53:26,727 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T11:53:26,728 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for f47e749e6316d0ed90d8985c2e855dcb in 873ms, sequenceid=195, compaction requested=true 2024-11-15T11:53:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T11:53:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T11:53:26,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T11:53:26,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-15T11:53:26,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0310 sec 2024-11-15T11:53:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:26,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:26,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:26,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7ec703faeceb48419213fc6dff821c5f is 50, key is test_row_0/A:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:26,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742315_1491 (size=16931) 2024-11-15T11:53:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T11:53:26,843 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:26,844 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T11:53:26,846 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:26,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:26,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:26,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671606836, max timestamp=1731671606857Num uniques 859; 2024-11-15T11:53:26,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:26,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671606836, max timestamp=1731671606857Num uniques 859; 2024-11-15T11:53:26,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671606836, max timestamp=1731671606857Num uniques 859; 2024-11-15T11:53:26,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:26,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:26,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671666896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:26,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671666904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671666908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671666904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:26,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671666918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:26,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:26,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671667022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671667042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671667042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671667042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671667051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:27,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:27,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:27,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7ec703faeceb48419213fc6dff821c5f 2024-11-15T11:53:27,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/3edbd31f4544461cb971dde5ec6aa95f is 50, key is test_row_0/B:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:27,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742316_1492 (size=12151) 2024-11-15T11:53:27,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/3edbd31f4544461cb971dde5ec6aa95f 2024-11-15T11:53:27,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/eda1117c4bcc416db9bbe6a2620b7d9b is 50, key is test_row_0/C:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742317_1493 (size=12151) 2024-11-15T11:53:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671667240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671667260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671667261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671667261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671667270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:27,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671667560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671667571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671667571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671667571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671667590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:27,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:27,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:27,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:27,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/eda1117c4bcc416db9bbe6a2620b7d9b 2024-11-15T11:53:27,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7ec703faeceb48419213fc6dff821c5f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f 2024-11-15T11:53:27,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f, entries=250, sequenceid=205, filesize=16.5 K 2024-11-15T11:53:27,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/3edbd31f4544461cb971dde5ec6aa95f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f 2024-11-15T11:53:27,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T11:53:27,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/eda1117c4bcc416db9bbe6a2620b7d9b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b 2024-11-15T11:53:27,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b, entries=150, sequenceid=205, filesize=11.9 K 2024-11-15T11:53:27,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f47e749e6316d0ed90d8985c2e855dcb in 852ms, sequenceid=205, compaction requested=true 2024-11-15T11:53:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:27,631 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:27,632 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:27,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:27,632 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:27,632 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:27,632 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,632 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ae0cc90cbeac429aaaf98274d9570db1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=54.7 K 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0cc90cbeac429aaaf98274d9570db1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603018 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:27,633 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,633 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/0a61a548d1f34106a01d5cdc0cdfc942, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=47.7 K 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ce1b5a013594a34b126e9d819583cff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731671604236 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a61a548d1f34106a01d5cdc0cdfc942, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603018 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting aabfa3075aff40ae8fd3a3c0a2b2af4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671604587 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e0cdb6844644f493a7085709575fb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731671604237 2024-11-15T11:53:27,633 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec703faeceb48419213fc6dff821c5f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606772 2024-11-15T11:53:27,634 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b0278442bdb047bd87d5af09d62dc1a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671604587 2024-11-15T11:53:27,634 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edbd31f4544461cb971dde5ec6aa95f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606775 2024-11-15T11:53:27,641 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:27,642 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/c8c633069e6244f695225d846045f6c9 is 50, key is test_row_0/A:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:27,645 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#418 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:27,646 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/465634307b9844fd80d79c8dfaf39b37 is 50, key is test_row_0/B:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742318_1494 (size=12527) 2024-11-15T11:53:27,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742319_1495 (size=12527) 2024-11-15T11:53:27,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T11:53:27,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:27,763 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:27,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cc9c03671cab4110880a047ccb601f2e is 50, key is test_row_0/A:col10/1731671606904/Put/seqid=0 2024-11-15T11:53:27,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742320_1496 (size=12151) 2024-11-15T11:53:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:28,056 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/c8c633069e6244f695225d846045f6c9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/c8c633069e6244f695225d846045f6c9 2024-11-15T11:53:28,060 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/465634307b9844fd80d79c8dfaf39b37 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/465634307b9844fd80d79c8dfaf39b37 2024-11-15T11:53:28,061 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into c8c633069e6244f695225d846045f6c9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:28,061 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,061 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=12, startTime=1731671607631; duration=0sec 2024-11-15T11:53:28,061 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:28,061 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:28,061 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:28,062 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:28,062 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:28,062 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:28,062 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef30ba4db6f44afb8f7f81d5b4b7b931, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=47.7 K 2024-11-15T11:53:28,063 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef30ba4db6f44afb8f7f81d5b4b7b931, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731671603018 2024-11-15T11:53:28,063 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31a0621930e243b98ad365d052f540a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731671604237 2024-11-15T11:53:28,063 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98c96b0d3a0484c891fc770f7449f7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731671604587 2024-11-15T11:53:28,063 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda1117c4bcc416db9bbe6a2620b7d9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606775 2024-11-15T11:53:28,064 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into 465634307b9844fd80d79c8dfaf39b37(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:28,064 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,064 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=12, startTime=1731671607632; duration=0sec 2024-11-15T11:53:28,064 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:28,064 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:28,070 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:28,071 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/7ff39d63002e416591e44942d251ba6b is 50, key is test_row_0/C:col10/1731671606775/Put/seqid=0 2024-11-15T11:53:28,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742321_1497 (size=12527) 2024-11-15T11:53:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:28,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671668098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671668099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671668100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671668110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,172 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cc9c03671cab4110880a047ccb601f2e 2024-11-15T11:53:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/22e2cd663ef24aba9d74137823bc2fa3 is 50, key is test_row_0/B:col10/1731671606904/Put/seqid=0 2024-11-15T11:53:28,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742322_1498 (size=12151) 2024-11-15T11:53:28,186 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/22e2cd663ef24aba9d74137823bc2fa3 2024-11-15T11:53:28,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5c091ed9ab364adb8d2a73bccf90f555 is 50, key is test_row_0/C:col10/1731671606904/Put/seqid=0 2024-11-15T11:53:28,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742323_1499 (size=12151) 2024-11-15T11:53:28,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5c091ed9ab364adb8d2a73bccf90f555 2024-11-15T11:53:28,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cc9c03671cab4110880a047ccb601f2e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e 2024-11-15T11:53:28,213 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T11:53:28,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/22e2cd663ef24aba9d74137823bc2fa3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3 2024-11-15T11:53:28,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T11:53:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5c091ed9ab364adb8d2a73bccf90f555 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555 2024-11-15T11:53:28,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671668211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,224 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555, entries=150, sequenceid=242, filesize=11.9 K 2024-11-15T11:53:28,225 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f47e749e6316d0ed90d8985c2e855dcb in 461ms, sequenceid=242, compaction requested=false 2024-11-15T11:53:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:28,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T11:53:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T11:53:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:28,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:28,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T11:53:28,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3790 sec 2024-11-15T11:53:28,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3870 sec 2024-11-15T11:53:28,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/53023d19b5694ef9861c1cb57428f5ca is 50, key is test_row_0/A:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742324_1500 (size=14541) 2024-11-15T11:53:28,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/53023d19b5694ef9861c1cb57428f5ca 2024-11-15T11:53:28,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/775aca4db0334a38b3eaa08265654f26 is 50, key is test_row_0/B:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742325_1501 (size=12151) 2024-11-15T11:53:28,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671608303, max timestamp=1731671608346Num uniques 859; 2024-11-15T11:53:28,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:28,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671608303, max timestamp=1731671608346Num uniques 859; 2024-11-15T11:53:28,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:28,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671608303, max timestamp=1731671608346Num uniques 859; 2024-11-15T11:53:28,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:28,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671668430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671668438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671668438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,480 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/7ff39d63002e416591e44942d251ba6b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ff39d63002e416591e44942d251ba6b 2024-11-15T11:53:28,484 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into 7ff39d63002e416591e44942d251ba6b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:28,484 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,484 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=12, startTime=1731671607632; duration=0sec 2024-11-15T11:53:28,484 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:28,484 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:28,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671668560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671668561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/775aca4db0334a38b3eaa08265654f26 2024-11-15T11:53:28,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/c5ce7508464c4fd0ab57245544ec8eca is 50, key is test_row_0/C:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742326_1502 (size=12151) 2024-11-15T11:53:28,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/c5ce7508464c4fd0ab57245544ec8eca 2024-11-15T11:53:28,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/53023d19b5694ef9861c1cb57428f5ca as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca 2024-11-15T11:53:28,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca, entries=200, sequenceid=255, filesize=14.2 K 2024-11-15T11:53:28,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/775aca4db0334a38b3eaa08265654f26 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26 2024-11-15T11:53:28,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26, entries=150, sequenceid=255, filesize=11.9 K 2024-11-15T11:53:28,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/c5ce7508464c4fd0ab57245544ec8eca as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca 2024-11-15T11:53:28,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca, entries=150, sequenceid=255, filesize=11.9 K 2024-11-15T11:53:28,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f47e749e6316d0ed90d8985c2e855dcb in 483ms, sequenceid=255, compaction requested=true 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:28,712 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:28,712 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:28,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:28,713 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:28,713 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:28,713 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:28,713 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/c8c633069e6244f695225d846045f6c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=38.3 K 2024-11-15T11:53:28,713 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:28,713 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:28,713 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:28,713 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/465634307b9844fd80d79c8dfaf39b37, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.0 K 2024-11-15T11:53:28,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 465634307b9844fd80d79c8dfaf39b37, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606775 2024-11-15T11:53:28,715 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c633069e6244f695225d846045f6c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606775 2024-11-15T11:53:28,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e2cd663ef24aba9d74137823bc2fa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731671606898 2024-11-15T11:53:28,716 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc9c03671cab4110880a047ccb601f2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731671606898 2024-11-15T11:53:28,716 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 775aca4db0334a38b3eaa08265654f26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608098 2024-11-15T11:53:28,716 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53023d19b5694ef9861c1cb57428f5ca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608097 2024-11-15T11:53:28,723 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:28,723 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:28,724 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8903ee165606489a9b5d3a6ca571312a is 50, key is test_row_0/A:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,724 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/229aec5ddd034f78a087c376a903a685 is 50, key is test_row_0/B:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T11:53:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:28,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:28,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:28,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742327_1503 (size=12629) 2024-11-15T11:53:28,787 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/229aec5ddd034f78a087c376a903a685 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/229aec5ddd034f78a087c376a903a685 2024-11-15T11:53:28,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7e83515b30084b219979a345b65df9dd is 50, key is test_row_0/A:col10/1731671608439/Put/seqid=0 2024-11-15T11:53:28,792 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into 229aec5ddd034f78a087c376a903a685(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:28,792 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:28,792 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=13, startTime=1731671608712; duration=0sec 2024-11-15T11:53:28,793 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:28,793 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:28,793 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742328_1504 (size=12629) 2024-11-15T11:53:28,794 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:28,795 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:28,795 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:28,795 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ff39d63002e416591e44942d251ba6b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.0 K 2024-11-15T11:53:28,796 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ff39d63002e416591e44942d251ba6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731671606775 2024-11-15T11:53:28,797 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c091ed9ab364adb8d2a73bccf90f555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731671606898 2024-11-15T11:53:28,797 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ce7508464c4fd0ab57245544ec8eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608098 2024-11-15T11:53:28,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671668803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742329_1505 (size=14741) 2024-11-15T11:53:28,827 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:28,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,827 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/be506b77658848f9a04f9b86d55a5d24 is 50, key is test_row_0/C:col10/1731671608098/Put/seqid=0 2024-11-15T11:53:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742330_1506 (size=12629) 2024-11-15T11:53:28,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671668921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671668930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671668940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671668940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T11:53:28,990 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:28,991 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T11:53:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:28,992 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:28,993 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:28,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:29,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671669130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,134 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T11:53:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:29,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:29,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671669140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671669141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671669160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671669160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,198 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8903ee165606489a9b5d3a6ca571312a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8903ee165606489a9b5d3a6ca571312a 2024-11-15T11:53:29,204 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into 8903ee165606489a9b5d3a6ca571312a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:29,204 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:29,204 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=13, startTime=1731671608712; duration=0sec 2024-11-15T11:53:29,204 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:29,204 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:29,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7e83515b30084b219979a345b65df9dd 2024-11-15T11:53:29,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/37e631779f9a44f2b4bdede6137fa6f9 is 50, key is test_row_0/B:col10/1731671608439/Put/seqid=0 2024-11-15T11:53:29,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742331_1507 (size=12301) 2024-11-15T11:53:29,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/37e631779f9a44f2b4bdede6137fa6f9 2024-11-15T11:53:29,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/b2ab5394e609442ca2c32e4714231469 is 50, key is test_row_0/C:col10/1731671608439/Put/seqid=0 2024-11-15T11:53:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742332_1508 (size=12301) 2024-11-15T11:53:29,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/b2ab5394e609442ca2c32e4714231469 2024-11-15T11:53:29,277 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/be506b77658848f9a04f9b86d55a5d24 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/be506b77658848f9a04f9b86d55a5d24 2024-11-15T11:53:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/7e83515b30084b219979a345b65df9dd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd 2024-11-15T11:53:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd, entries=200, sequenceid=292, filesize=14.4 K 2024-11-15T11:53:29,281 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into be506b77658848f9a04f9b86d55a5d24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:29,281 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:29,281 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=13, startTime=1731671608712; duration=0sec 2024-11-15T11:53:29,281 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:29,281 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/37e631779f9a44f2b4bdede6137fa6f9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9 2024-11-15T11:53:29,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T11:53:29,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/b2ab5394e609442ca2c32e4714231469 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469 2024-11-15T11:53:29,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469, entries=150, sequenceid=292, filesize=12.0 K 2024-11-15T11:53:29,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f47e749e6316d0ed90d8985c2e855dcb in 527ms, sequenceid=292, compaction requested=false 2024-11-15T11:53:29,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:29,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:29,299 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:29,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cf2adbf39a984775a07b137409f2f4ad is 50, key is test_row_0/A:col10/1731671608804/Put/seqid=0 2024-11-15T11:53:29,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742333_1509 (size=12301) 2024-11-15T11:53:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:29,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671609504, max timestamp=1731671609519Num uniques 859; 2024-11-15T11:53:29,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:29,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671609504, max timestamp=1731671609519Num uniques 859; 2024-11-15T11:53:29,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:29,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671609504, max timestamp=1731671609519Num uniques 859; 2024-11-15T11:53:29,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671669580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671669589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671669591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671669593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:29,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671669701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671669701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,713 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cf2adbf39a984775a07b137409f2f4ad 2024-11-15T11:53:29,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/f702b085911149cd9eda9dcbc67145b0 is 50, key is test_row_0/B:col10/1731671608804/Put/seqid=0 2024-11-15T11:53:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671669720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671669721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742334_1510 (size=12301) 2024-11-15T11:53:29,749 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/f702b085911149cd9eda9dcbc67145b0 2024-11-15T11:53:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fa478cc4577647a1a54cd4f1158c8747 is 50, key is test_row_0/C:col10/1731671608804/Put/seqid=0 2024-11-15T11:53:29,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742335_1511 (size=12301) 2024-11-15T11:53:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671669910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671669911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671669931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671669931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:30,205 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fa478cc4577647a1a54cd4f1158c8747 2024-11-15T11:53:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/cf2adbf39a984775a07b137409f2f4ad as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad 2024-11-15T11:53:30,226 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad, entries=150, sequenceid=305, filesize=12.0 K 2024-11-15T11:53:30,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/f702b085911149cd9eda9dcbc67145b0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0 2024-11-15T11:53:30,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0, entries=150, sequenceid=305, filesize=12.0 K 2024-11-15T11:53:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fa478cc4577647a1a54cd4f1158c8747 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747 2024-11-15T11:53:30,240 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747, entries=150, sequenceid=305, filesize=12.0 K 2024-11-15T11:53:30,242 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f47e749e6316d0ed90d8985c2e855dcb in 942ms, sequenceid=305, compaction requested=true 2024-11-15T11:53:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T11:53:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T11:53:30,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T11:53:30,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-11-15T11:53:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:30,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-15T11:53:30,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:30,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:30,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:30,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2550 sec 2024-11-15T11:53:30,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e2c127617fe34c24b38bae978add5d6d is 50, key is test_row_0/A:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742336_1512 (size=17181) 2024-11-15T11:53:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671670266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671670268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671670272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671670273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671670380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671670390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671670391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671670391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671670600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671670611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671670620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671670620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e2c127617fe34c24b38bae978add5d6d 2024-11-15T11:53:30,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/9f612cc4f37743d6a10a29f3b4c9e3bb is 50, key is test_row_0/B:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742337_1513 (size=12301) 2024-11-15T11:53:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671670921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671670930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671670940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671670941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/9f612cc4f37743d6a10a29f3b4c9e3bb 2024-11-15T11:53:31,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f73d567244904d61b208b1f3ae3bedcd is 50, key is test_row_0/C:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742338_1514 (size=12301) 2024-11-15T11:53:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T11:53:31,141 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:31,141 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T11:53:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:31,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:31,144 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:31,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671671150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:31,168 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:31,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T11:53:31,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:31,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671671440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T11:53:31,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:31,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671671460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671671460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671671461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:31,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f73d567244904d61b208b1f3ae3bedcd 2024-11-15T11:53:31,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e2c127617fe34c24b38bae978add5d6d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d 2024-11-15T11:53:31,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d, entries=250, sequenceid=344, filesize=16.8 K 2024-11-15T11:53:31,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/9f612cc4f37743d6a10a29f3b4c9e3bb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb 2024-11-15T11:53:31,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T11:53:31,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/f73d567244904d61b208b1f3ae3bedcd as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd 2024-11-15T11:53:31,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd, entries=150, sequenceid=344, filesize=12.0 K 2024-11-15T11:53:31,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for f47e749e6316d0ed90d8985c2e855dcb in 1267ms, sequenceid=344, compaction requested=true 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:31,513 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:31,513 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:31,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:31,522 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,522 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,522 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8903ee165606489a9b5d3a6ca571312a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=55.5 K 2024-11-15T11:53:31,522 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/229aec5ddd034f78a087c376a903a685, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=48.4 K 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8903ee165606489a9b5d3a6ca571312a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608098 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 229aec5ddd034f78a087c376a903a685, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608098 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e83515b30084b219979a345b65df9dd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731671608414 2024-11-15T11:53:31,522 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 37e631779f9a44f2b4bdede6137fa6f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731671608435 2024-11-15T11:53:31,523 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2adbf39a984775a07b137409f2f4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731671608801 2024-11-15T11:53:31,523 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f702b085911149cd9eda9dcbc67145b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731671608801 2024-11-15T11:53:31,523 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c127617fe34c24b38bae978add5d6d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609585 2024-11-15T11:53:31,523 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f612cc4f37743d6a10a29f3b4c9e3bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609586 2024-11-15T11:53:31,539 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:31,540 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/e4f0e8d157084ea58a94d892c8050354 is 50, key is test_row_0/B:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:31,540 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:31,541 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/9c8f27d7a5de47ad8c6d62d313e4018b is 50, key is test_row_0/A:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:31,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742340_1516 (size=12915) 2024-11-15T11:53:31,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742339_1515 (size=12915) 2024-11-15T11:53:31,564 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/e4f0e8d157084ea58a94d892c8050354 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e4f0e8d157084ea58a94d892c8050354 2024-11-15T11:53:31,567 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into e4f0e8d157084ea58a94d892c8050354(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:31,567 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:31,567 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=12, startTime=1731671611513; duration=0sec 2024-11-15T11:53:31,568 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:31,568 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:31,568 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:31,568 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:31,568 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:31,569 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,569 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/be506b77658848f9a04f9b86d55a5d24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=48.4 K 2024-11-15T11:53:31,569 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting be506b77658848f9a04f9b86d55a5d24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731671608098 2024-11-15T11:53:31,569 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ab5394e609442ca2c32e4714231469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731671608435 2024-11-15T11:53:31,569 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fa478cc4577647a1a54cd4f1158c8747, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731671608801 2024-11-15T11:53:31,569 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f73d567244904d61b208b1f3ae3bedcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609586 2024-11-15T11:53:31,576 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:31,577 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fe756b72dfc345fe89843ec48b6e1654 is 50, key is test_row_0/C:col10/1731671609591/Put/seqid=0 2024-11-15T11:53:31,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742341_1517 (size=12915) 2024-11-15T11:53:31,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T11:53:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:31,606 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:53:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:31,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:31,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/b4b8329bb56b4fd3a231551c1365c7e1 is 50, key is test_row_0/A:col10/1731671610261/Put/seqid=0 2024-11-15T11:53:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742342_1518 (size=12301) 2024-11-15T11:53:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:31,953 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/9c8f27d7a5de47ad8c6d62d313e4018b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/9c8f27d7a5de47ad8c6d62d313e4018b 2024-11-15T11:53:31,957 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into 9c8f27d7a5de47ad8c6d62d313e4018b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:31,957 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:31,957 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=12, startTime=1731671611513; duration=0sec 2024-11-15T11:53:31,957 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:31,957 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:31,986 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fe756b72dfc345fe89843ec48b6e1654 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fe756b72dfc345fe89843ec48b6e1654 2024-11-15T11:53:31,989 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into fe756b72dfc345fe89843ec48b6e1654(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:31,989 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:31,990 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=12, startTime=1731671611513; duration=0sec 2024-11-15T11:53:31,990 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:31,990 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:32,017 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/b4b8329bb56b4fd3a231551c1365c7e1 2024-11-15T11:53:32,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/d5af95392d394e759d14cfef06a808c4 is 50, key is test_row_0/B:col10/1731671610261/Put/seqid=0 2024-11-15T11:53:32,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742343_1519 (size=12301) 2024-11-15T11:53:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:32,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/d5af95392d394e759d14cfef06a808c4 2024-11-15T11:53:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ff373d1cfb5349e1bc99357bea26b168 is 50, key is test_row_0/C:col10/1731671610261/Put/seqid=0 2024-11-15T11:53:32,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742344_1520 (size=12301) 2024-11-15T11:53:32,445 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ff373d1cfb5349e1bc99357bea26b168 2024-11-15T11:53:32,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/b4b8329bb56b4fd3a231551c1365c7e1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1 2024-11-15T11:53:32,451 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T11:53:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/d5af95392d394e759d14cfef06a808c4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4 2024-11-15T11:53:32,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T11:53:32,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ff373d1cfb5349e1bc99357bea26b168 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168 2024-11-15T11:53:32,459 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168, entries=150, sequenceid=353, filesize=12.0 K 2024-11-15T11:53:32,460 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f47e749e6316d0ed90d8985c2e855dcb in 854ms, sequenceid=353, compaction requested=false 2024-11-15T11:53:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T11:53:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T11:53:32,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T11:53:32,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-11-15T11:53:32,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3240 sec 2024-11-15T11:53:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:32,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:32,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/f0f3d48c29734647b14c9cf6d2926cfe is 50, key is test_row_0/A:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742345_1521 (size=12301) 2024-11-15T11:53:32,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671612552, max timestamp=1731671612565Num uniques 859; 2024-11-15T11:53:32,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:32,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671672570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671672571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671672578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671672580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671672691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671672691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671672691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671672700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671672901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/f0f3d48c29734647b14c9cf6d2926cfe 2024-11-15T11:53:32,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/e42d35b572164382b42bff396867b83b is 50, key is test_row_0/B:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:32,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742346_1522 (size=12301) 2024-11-15T11:53:32,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671672910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671672910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671672910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671673210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671673231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671673231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671673231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T11:53:33,291 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:33,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T11:53:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:33,292 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:33,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:33,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:33,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/e42d35b572164382b42bff396867b83b 2024-11-15T11:53:33,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a96da9492ea04a6abe64bc8a0e2bc9f4 is 50, key is test_row_0/C:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:33,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742347_1523 (size=12301) 2024-11-15T11:53:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:33,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:33,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:33,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:33,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a96da9492ea04a6abe64bc8a0e2bc9f4 2024-11-15T11:53:33,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/f0f3d48c29734647b14c9cf6d2926cfe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe 2024-11-15T11:53:33,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T11:53:33,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/e42d35b572164382b42bff396867b83b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b 2024-11-15T11:53:33,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671673730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T11:53:33,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/a96da9492ea04a6abe64bc8a0e2bc9f4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4 2024-11-15T11:53:33,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T11:53:33,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for f47e749e6316d0ed90d8985c2e855dcb in 1248ms, sequenceid=365, compaction requested=true 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:33,741 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:33,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:33,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:33,742 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,742 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,742 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e4f0e8d157084ea58a94d892c8050354, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.6 K 2024-11-15T11:53:33,742 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/9c8f27d7a5de47ad8c6d62d313e4018b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.6 K 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f0e8d157084ea58a94d892c8050354, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609586 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c8f27d7a5de47ad8c6d62d313e4018b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609586 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d5af95392d394e759d14cfef06a808c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731671610261 2024-11-15T11:53:33,742 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4b8329bb56b4fd3a231551c1365c7e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731671610261 2024-11-15T11:53:33,743 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e42d35b572164382b42bff396867b83b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:33,743 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f3d48c29734647b14c9cf6d2926cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:33,747 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:33,748 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/2cd9611f911c4dd8adf5b94c2e23eb5b is 50, key is test_row_0/A:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:33,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=134.18 KB heapSize=391.80 KB 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:33,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:33,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:33,750 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,751 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ba3b801d525f4579ad3f8bb4df3cf686 is 50, key is test_row_0/B:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e571937239e54d41be70e57d7c730d52 is 50, key is test_row_0/A:col10/1731671612552/Put/seqid=0 2024-11-15T11:53:33,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742348_1524 (size=13017) 2024-11-15T11:53:33,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742349_1525 (size=13017) 2024-11-15T11:53:33,766 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/2cd9611f911c4dd8adf5b94c2e23eb5b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/2cd9611f911c4dd8adf5b94c2e23eb5b 2024-11-15T11:53:33,770 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into 2cd9611f911c4dd8adf5b94c2e23eb5b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:33,771 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:33,771 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=13, startTime=1731671613741; duration=0sec 2024-11-15T11:53:33,771 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:33,771 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:33,771 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:33,773 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:33,773 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:33,774 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,774 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fe756b72dfc345fe89843ec48b6e1654, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.6 K 2024-11-15T11:53:33,774 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe756b72dfc345fe89843ec48b6e1654, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731671609586 2024-11-15T11:53:33,774 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff373d1cfb5349e1bc99357bea26b168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731671610261 2024-11-15T11:53:33,774 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96da9492ea04a6abe64bc8a0e2bc9f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:33,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742350_1526 (size=14741) 2024-11-15T11:53:33,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e571937239e54d41be70e57d7c730d52 2024-11-15T11:53:33,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671673763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,781 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:33,782 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/c024da49b64a496cafa111a214de0e4a is 50, key is test_row_0/C:col10/1731671612480/Put/seqid=0 2024-11-15T11:53:33,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/c3fd4dab3ece4bdba0c671cc6c18e9df is 50, key is test_row_0/B:col10/1731671612552/Put/seqid=0 2024-11-15T11:53:33,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671673770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671673772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742351_1527 (size=12301) 2024-11-15T11:53:33,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742352_1528 (size=13017) 2024-11-15T11:53:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671673880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671673891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671673891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:33,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:33,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:33,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:33,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:33,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:34,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671674091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671674100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671674100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,169 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ba3b801d525f4579ad3f8bb4df3cf686 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ba3b801d525f4579ad3f8bb4df3cf686 2024-11-15T11:53:34,172 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into ba3b801d525f4579ad3f8bb4df3cf686(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:34,172 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:34,173 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=13, startTime=1731671613741; duration=0sec 2024-11-15T11:53:34,173 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:34,173 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:34,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/c3fd4dab3ece4bdba0c671cc6c18e9df 2024-11-15T11:53:34,199 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/c024da49b64a496cafa111a214de0e4a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c024da49b64a496cafa111a214de0e4a 2024-11-15T11:53:34,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ef0ae66db94743d1a482008fb7b9db72 is 50, key is test_row_0/C:col10/1731671612552/Put/seqid=0 2024-11-15T11:53:34,203 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into c024da49b64a496cafa111a214de0e4a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:34,203 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:34,203 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=13, startTime=1731671613741; duration=0sec 2024-11-15T11:53:34,203 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:34,203 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:34,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:34,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742353_1529 (size=12301) 2024-11-15T11:53:34,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671674411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671674411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671674420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:34,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:34,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ef0ae66db94743d1a482008fb7b9db72 2024-11-15T11:53:34,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e571937239e54d41be70e57d7c730d52 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52 2024-11-15T11:53:34,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52, entries=200, sequenceid=392, filesize=14.4 K 2024-11-15T11:53:34,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/c3fd4dab3ece4bdba0c671cc6c18e9df as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df 2024-11-15T11:53:34,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T11:53:34,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/ef0ae66db94743d1a482008fb7b9db72 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72 2024-11-15T11:53:34,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T11:53:34,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~391.75 KB/401152, currentSize=40.25 KB/41220 for f47e749e6316d0ed90d8985c2e855dcb in 883ms, sequenceid=392, compaction requested=false 2024-11-15T11:53:34,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:34,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T11:53:34,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:34,666 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:34,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d0e8807faca344c2a3f5a134dc6b7b38 is 50, key is test_row_0/A:col10/1731671613766/Put/seqid=0 2024-11-15T11:53:34,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742354_1530 (size=12301) 2024-11-15T11:53:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:34,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671614819, max timestamp=1731671614842Num uniques 859; 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671614819, max timestamp=1731671614842Num uniques 859; 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671614819, max timestamp=1731671614842Num uniques 859; 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:34,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671674931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671674940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671674940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671674941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:34,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671675050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671675060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671675060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d0e8807faca344c2a3f5a134dc6b7b38 2024-11-15T11:53:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ae6db74f4bfc45f2b979cfe14fd80188 is 50, key is test_row_0/B:col10/1731671613766/Put/seqid=0 2024-11-15T11:53:35,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742355_1531 (size=12301) 2024-11-15T11:53:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34394 deadline: 1731671675200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,210 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671675260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671675280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671675280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ae6db74f4bfc45f2b979cfe14fd80188 2024-11-15T11:53:35,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/3bb4385d1ac84124bba5629e0d4cadb9 is 50, key is test_row_0/C:col10/1731671613766/Put/seqid=0 2024-11-15T11:53:35,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742356_1532 (size=12301) 2024-11-15T11:53:35,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671675581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671675590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671675590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:35,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/3bb4385d1ac84124bba5629e0d4cadb9 2024-11-15T11:53:35,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d0e8807faca344c2a3f5a134dc6b7b38 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38 2024-11-15T11:53:35,895 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T11:53:35,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/ae6db74f4bfc45f2b979cfe14fd80188 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188 2024-11-15T11:53:35,898 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T11:53:35,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/3bb4385d1ac84124bba5629e0d4cadb9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9 2024-11-15T11:53:35,901 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T11:53:35,901 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f47e749e6316d0ed90d8985c2e855dcb in 1235ms, sequenceid=404, compaction requested=true 2024-11-15T11:53:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T11:53:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T11:53:35,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T11:53:35,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6100 sec 2024-11-15T11:53:35,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.6130 sec 2024-11-15T11:53:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:35,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-15T11:53:35,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6a625f0450884b3ea9f9af82630f693c is 50, key is test_row_0/A:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:35,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742357_1533 (size=14741) 2024-11-15T11:53:36,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671676012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671676101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671676110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671676110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671676130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671676351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6a625f0450884b3ea9f9af82630f693c 2024-11-15T11:53:36,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/5b2ef90cc6954dca9b3ce55a00d12014 is 50, key is test_row_0/B:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:36,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742358_1534 (size=12301) 2024-11-15T11:53:36,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671676670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:36,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:36,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/5b2ef90cc6954dca9b3ce55a00d12014 2024-11-15T11:53:36,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/d0a8ca890ade4f8ba9d25aeff0a72e59 is 50, key is test_row_0/C:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:36,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742359_1535 (size=12301) 2024-11-15T11:53:37,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671677120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:37,127 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:37,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:37,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671677130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671677130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:37,140 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,140 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:37,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:37,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671677180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:37,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/d0a8ca890ade4f8ba9d25aeff0a72e59 2024-11-15T11:53:37,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/6a625f0450884b3ea9f9af82630f693c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c 2024-11-15T11:53:37,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c, entries=200, sequenceid=442, filesize=14.4 K 2024-11-15T11:53:37,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/5b2ef90cc6954dca9b3ce55a00d12014 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014 2024-11-15T11:53:37,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T11:53:37,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/d0a8ca890ade4f8ba9d25aeff0a72e59 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59 2024-11-15T11:53:37,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T11:53:37,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for f47e749e6316d0ed90d8985c2e855dcb in 1275ms, sequenceid=442, compaction requested=true 2024-11-15T11:53:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:37,243 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:37,244 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:37,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:37,244 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:37,245 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:37,245 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/2cd9611f911c4dd8adf5b94c2e23eb5b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=53.5 K 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd9611f911c4dd8adf5b94c2e23eb5b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:37,245 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:37,245 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ba3b801d525f4579ad3f8bb4df3cf686, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=48.8 K 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e571937239e54d41be70e57d7c730d52, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731671612552 2024-11-15T11:53:37,245 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3b801d525f4579ad3f8bb4df3cf686, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:37,246 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c3fd4dab3ece4bdba0c671cc6c18e9df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731671612552 2024-11-15T11:53:37,246 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e8807faca344c2a3f5a134dc6b7b38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731671613760 2024-11-15T11:53:37,246 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6db74f4bfc45f2b979cfe14fd80188, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731671613760 2024-11-15T11:53:37,246 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a625f0450884b3ea9f9af82630f693c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:37,246 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2ef90cc6954dca9b3ce55a00d12014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:37,251 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:37,252 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:37,253 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/cbfdeab117204f928c9aad4a9636a670 is 50, key is test_row_0/B:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:37,253 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d535c0a7c88846daad9c8dd2e06e3fff is 50, key is test_row_0/A:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:37,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742360_1536 (size=13153) 2024-11-15T11:53:37,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742361_1537 (size=13153) 2024-11-15T11:53:37,265 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/cbfdeab117204f928c9aad4a9636a670 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/cbfdeab117204f928c9aad4a9636a670 2024-11-15T11:53:37,270 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d535c0a7c88846daad9c8dd2e06e3fff as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d535c0a7c88846daad9c8dd2e06e3fff 2024-11-15T11:53:37,273 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into cbfdeab117204f928c9aad4a9636a670(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:37,273 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=12, startTime=1731671617243; duration=0sec 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:53:37,273 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into d535c0a7c88846daad9c8dd2e06e3fff(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:37,273 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=12, startTime=1731671617243; duration=0sec 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:37,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:37,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:53:37,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:37,274 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:37,274 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c024da49b64a496cafa111a214de0e4a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=48.8 K 2024-11-15T11:53:37,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c024da49b64a496cafa111a214de0e4a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731671612461 2024-11-15T11:53:37,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0ae66db94743d1a482008fb7b9db72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731671612552 2024-11-15T11:53:37,275 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb4385d1ac84124bba5629e0d4cadb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731671613760 2024-11-15T11:53:37,275 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a8ca890ade4f8ba9d25aeff0a72e59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:37,281 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:37,282 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/1457270fa2424fe0a2f6351fd5995b10 is 50, key is test_row_0/C:col10/1731671615966/Put/seqid=0 2024-11-15T11:53:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742362_1538 (size=13153) 2024-11-15T11:53:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T11:53:37,460 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:37,461 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T11:53:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:37,462 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:37,463 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:37,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:37,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:37,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:37,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:37,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8972bde9062545e69c0e2c2d1434495e is 50, key is test_row_0/A:col10/1731671616010/Put/seqid=0 2024-11-15T11:53:37,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742363_1539 (size=12301) 2024-11-15T11:53:37,688 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/1457270fa2424fe0a2f6351fd5995b10 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/1457270fa2424fe0a2f6351fd5995b10 2024-11-15T11:53:37,691 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into 1457270fa2424fe0a2f6351fd5995b10(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:37,692 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:37,692 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=12, startTime=1731671617244; duration=0sec 2024-11-15T11:53:37,692 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:37,692 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:38,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8972bde9062545e69c0e2c2d1434495e 2024-11-15T11:53:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/029c2ddf220d44359b7185d1557ca62c is 50, key is test_row_0/B:col10/1731671616010/Put/seqid=0 2024-11-15T11:53:38,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742364_1540 (size=12301) 2024-11-15T11:53:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:38,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. as already flushing 2024-11-15T11:53:38,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671618244, max timestamp=1731671618311Num uniques 859; 2024-11-15T11:53:38,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:38,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671618244, max timestamp=1731671618311Num uniques 859; 2024-11-15T11:53:38,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:38,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671678415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:38,437 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/029c2ddf220d44359b7185d1557ca62c 2024-11-15T11:53:38,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/492faa5c41e8450e9ba80b187dd25408 is 50, key is test_row_0/C:col10/1731671616010/Put/seqid=0 2024-11-15T11:53:38,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742365_1541 (size=12301) 2024-11-15T11:53:38,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671678531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:38,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671678740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:38,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:38,845 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/492faa5c41e8450e9ba80b187dd25408 2024-11-15T11:53:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/8972bde9062545e69c0e2c2d1434495e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e 2024-11-15T11:53:38,850 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e, entries=150, sequenceid=452, filesize=12.0 K 2024-11-15T11:53:38,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/029c2ddf220d44359b7185d1557ca62c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c 2024-11-15T11:53:38,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c, entries=150, sequenceid=452, filesize=12.0 K 2024-11-15T11:53:38,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/492faa5c41e8450e9ba80b187dd25408 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408 2024-11-15T11:53:38,856 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408, entries=150, sequenceid=452, filesize=12.0 K 2024-11-15T11:53:38,856 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=145.36 KB/148850 for f47e749e6316d0ed90d8985c2e855dcb in 1241ms, sequenceid=452, compaction requested=false 2024-11-15T11:53:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:38,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T11:53:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T11:53:38,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T11:53:38,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-11-15T11:53:38,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3990 sec 2024-11-15T11:53:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:39,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=152.07 KB heapSize=443.08 KB 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:39,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:39,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d951c78f50c544e7814709dbba883977 is 50, key is test_row_0/A:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:39,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742366_1542 (size=14741) 2024-11-15T11:53:39,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671679085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34346 deadline: 1731671679130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,139 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34404 deadline: 1731671679160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34328 deadline: 1731671679160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,165 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_1' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,165 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in f47e749e6316d0ed90d8985c2e855dcb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671679201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,364 INFO [Thread-2068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2068. 2024-11-15T11:53:39,364 DEBUG [Thread-2068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:39,364 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,365 DEBUG [Thread-2068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:39,369 INFO [Thread-2066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2066. 2024-11-15T11:53:39,369 DEBUG [Thread-2066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:39,369 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,369 DEBUG [Thread-2066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:39,373 INFO [Thread-2072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2072. 2024-11-15T11:53:39,373 DEBUG [Thread-2072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:39,373 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,373 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:39,380 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-11-15T11:53:39,380 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:39,380 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,380 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:39,380 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-11-15T11:53:39,380 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:39,380 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,380 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:39,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671679410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d951c78f50c544e7814709dbba883977 2024-11-15T11:53:39,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/77d403f0f5644e1b9a9fef4ed6394711 is 50, key is test_row_0/B:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:39,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742367_1543 (size=12301) 2024-11-15T11:53:39,573 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:53:39,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T11:53:39,611 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:39,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671679721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:39,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:39,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/77d403f0f5644e1b9a9fef4ed6394711 2024-11-15T11:53:40,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5f80b10f75c3401993a9284a5735863f is 50, key is test_row_0/C:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:40,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742368_1544 (size=12301) 2024-11-15T11:53:40,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:40,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34356 deadline: 1731671680232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 , the old value is region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:40,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=2 because the exception is null or not the one we care about 2024-11-15T11:53:40,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5f80b10f75c3401993a9284a5735863f 2024-11-15T11:53:40,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/d951c78f50c544e7814709dbba883977 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977 2024-11-15T11:53:40,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977, entries=200, sequenceid=488, filesize=14.4 K 2024-11-15T11:53:40,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/77d403f0f5644e1b9a9fef4ed6394711 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711 2024-11-15T11:53:40,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711, entries=150, sequenceid=488, filesize=12.0 K 2024-11-15T11:53:40,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/5f80b10f75c3401993a9284a5735863f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f 2024-11-15T11:53:40,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f, entries=150, sequenceid=488, filesize=12.0 K 2024-11-15T11:53:40,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~443.03 KB/453664, currentSize=26.84 KB/27480 for f47e749e6316d0ed90d8985c2e855dcb in 1370ms, sequenceid=488, compaction requested=true 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:40,428 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f47e749e6316d0ed90d8985c2e855dcb:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:40,428 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/B is initiating minor compaction (all files) 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/A is initiating minor compaction (all files) 2024-11-15T11:53:40,429 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/A in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:40,429 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/B in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:40,429 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/cbfdeab117204f928c9aad4a9636a670, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.9 K 2024-11-15T11:53:40,429 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d535c0a7c88846daad9c8dd2e06e3fff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=39.3 K 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting cbfdeab117204f928c9aad4a9636a670, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:40,429 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d535c0a7c88846daad9c8dd2e06e3fff, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:40,430 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8972bde9062545e69c0e2c2d1434495e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731671615972 2024-11-15T11:53:40,430 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 029c2ddf220d44359b7185d1557ca62c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731671615972 2024-11-15T11:53:40,430 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d951c78f50c544e7814709dbba883977, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731671618375 2024-11-15T11:53:40,430 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d403f0f5644e1b9a9fef4ed6394711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731671618396 2024-11-15T11:53:40,435 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#A#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:40,435 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/fe87cb89bab443da95bf900ee67b17cf is 50, key is test_row_0/A:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:40,435 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:40,436 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/82594d58562a4aada2cf1dae4b7b1ede is 50, key is test_row_0/B:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742369_1545 (size=13255) 2024-11-15T11:53:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742370_1546 (size=13255) 2024-11-15T11:53:40,441 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/82594d58562a4aada2cf1dae4b7b1ede as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/82594d58562a4aada2cf1dae4b7b1ede 2024-11-15T11:53:40,444 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/B of f47e749e6316d0ed90d8985c2e855dcb into 82594d58562a4aada2cf1dae4b7b1ede(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:40,444 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:40,444 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/B, priority=13, startTime=1731671620428; duration=0sec 2024-11-15T11:53:40,444 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:40,444 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:B 2024-11-15T11:53:40,444 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:40,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:40,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): f47e749e6316d0ed90d8985c2e855dcb/C is initiating minor compaction (all files) 2024-11-15T11:53:40,445 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f47e749e6316d0ed90d8985c2e855dcb/C in TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:40,445 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/1457270fa2424fe0a2f6351fd5995b10, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp, totalSize=36.9 K 2024-11-15T11:53:40,445 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1457270fa2424fe0a2f6351fd5995b10, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731671614923 2024-11-15T11:53:40,446 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 492faa5c41e8450e9ba80b187dd25408, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731671615972 2024-11-15T11:53:40,446 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f80b10f75c3401993a9284a5735863f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731671618396 2024-11-15T11:53:40,450 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f47e749e6316d0ed90d8985c2e855dcb#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:40,450 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fb0463c523de43f6a92d03c708d18803 is 50, key is test_row_0/C:col10/1731671619057/Put/seqid=0 2024-11-15T11:53:40,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742371_1547 (size=13255) 2024-11-15T11:53:40,849 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/fe87cb89bab443da95bf900ee67b17cf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/fe87cb89bab443da95bf900ee67b17cf 2024-11-15T11:53:40,854 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/A of f47e749e6316d0ed90d8985c2e855dcb into fe87cb89bab443da95bf900ee67b17cf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:40,855 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:40,855 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/A, priority=13, startTime=1731671620428; duration=0sec 2024-11-15T11:53:40,855 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:40,855 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:A 2024-11-15T11:53:40,858 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/fb0463c523de43f6a92d03c708d18803 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fb0463c523de43f6a92d03c708d18803 2024-11-15T11:53:40,864 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f47e749e6316d0ed90d8985c2e855dcb/C of f47e749e6316d0ed90d8985c2e855dcb into fb0463c523de43f6a92d03c708d18803(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:40,864 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:40,864 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., storeName=f47e749e6316d0ed90d8985c2e855dcb/C, priority=13, startTime=1731671620428; duration=0sec 2024-11-15T11:53:40,865 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:40,865 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f47e749e6316d0ed90d8985c2e855dcb:C 2024-11-15T11:53:41,242 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-15T11:53:41,242 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:41,242 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:41,243 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:41,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:43,155 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-15T11:53:43,156 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:43,156 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,156 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:43,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:43,206 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-15T11:53:43,206 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:43,206 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:43,206 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:43,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:43,213 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-15T11:53:43,214 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:43,214 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,214 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:43,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:43,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e71bd08b96464890bb73458135d77779 is 50, key is test_row_0/A:col10/1731671619084/Put/seqid=0 2024-11-15T11:53:43,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742372_1548 (size=12301) 2024-11-15T11:53:43,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e71bd08b96464890bb73458135d77779 2024-11-15T11:53:43,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/68bd93fb3d894986a82967d15e58859c is 50, key is test_row_0/B:col10/1731671619084/Put/seqid=0 2024-11-15T11:53:43,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742373_1549 (size=12301) 2024-11-15T11:53:44,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/68bd93fb3d894986a82967d15e58859c 2024-11-15T11:53:44,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6854aac641a0426c99b241bc24da9e99 is 50, key is test_row_0/C:col10/1731671619084/Put/seqid=0 2024-11-15T11:53:44,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742374_1550 (size=12301) 2024-11-15T11:53:44,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6854aac641a0426c99b241bc24da9e99 2024-11-15T11:53:44,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/e71bd08b96464890bb73458135d77779 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e71bd08b96464890bb73458135d77779 2024-11-15T11:53:44,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e71bd08b96464890bb73458135d77779, entries=150, sequenceid=501, filesize=12.0 K 2024-11-15T11:53:44,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/68bd93fb3d894986a82967d15e58859c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/68bd93fb3d894986a82967d15e58859c 2024-11-15T11:53:44,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/68bd93fb3d894986a82967d15e58859c, entries=150, sequenceid=501, filesize=12.0 K 2024-11-15T11:53:44,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/6854aac641a0426c99b241bc24da9e99 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6854aac641a0426c99b241bc24da9e99 2024-11-15T11:53:44,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6854aac641a0426c99b241bc24da9e99, entries=150, sequenceid=501, filesize=12.0 K 2024-11-15T11:53:44,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for f47e749e6316d0ed90d8985c2e855dcb in 1271ms, sequenceid=501, compaction requested=false 2024-11-15T11:53:44,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f47e749e6316d0ed90d8985c2e855dcb: 2024-11-15T11:53:45,223 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-15T11:53:45,223 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:53:45,223 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:45,223 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:45,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:53:45,223 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1146 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3438 rows 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1153 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3450 rows 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1145 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3435 rows 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1150 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3450 rows 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-15T11:53:45,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3452 rows 2024-11-15T11:53:45,224 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:53:45,224 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:53:45,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:45,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:45,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:53:45,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:53:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:45,231 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671625230"}]},"ts":"1731671625230"} 2024-11-15T11:53:45,232 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:53:45,232 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:53:45,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:53:45,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, UNASSIGN}] 2024-11-15T11:53:45,236 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, UNASSIGN 2024-11-15T11:53:45,237 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f47e749e6316d0ed90d8985c2e855dcb, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:45,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, UNASSIGN because future has completed 2024-11-15T11:53:45,239 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:53:45,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:45,392 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:45,392 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:53:45,392 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing f47e749e6316d0ed90d8985c2e855dcb, disabling compactions & flushes 2024-11-15T11:53:45,393 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:45,393 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:45,393 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. after waiting 0 ms 2024-11-15T11:53:45,393 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:45,393 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing f47e749e6316d0ed90d8985c2e855dcb 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-15T11:53:45,393 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=A 2024-11-15T11:53:45,394 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:45,394 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=B 2024-11-15T11:53:45,394 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:45,394 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f47e749e6316d0ed90d8985c2e855dcb, store=C 2024-11-15T11:53:45,394 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:45,401 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/eed8995d28204b01a3959e53ac218411 is 50, key is test_row_0/A:col10/1731671623211/Put/seqid=0 2024-11-15T11:53:45,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742375_1551 (size=9857) 2024-11-15T11:53:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:45,805 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/eed8995d28204b01a3959e53ac218411 2024-11-15T11:53:45,815 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fe0e98083562429ab524721be2e012b1 is 50, key is test_row_0/B:col10/1731671623211/Put/seqid=0 2024-11-15T11:53:45,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742376_1552 (size=9857) 2024-11-15T11:53:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:46,219 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fe0e98083562429ab524721be2e012b1 2024-11-15T11:53:46,230 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/55d73757537b4ac3b21fe4b7970595f2 is 50, key is test_row_0/C:col10/1731671623211/Put/seqid=0 2024-11-15T11:53:46,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742377_1553 (size=9857) 2024-11-15T11:53:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:46,635 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/55d73757537b4ac3b21fe4b7970595f2 2024-11-15T11:53:46,642 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/A/eed8995d28204b01a3959e53ac218411 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/eed8995d28204b01a3959e53ac218411 2024-11-15T11:53:46,648 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/eed8995d28204b01a3959e53ac218411, entries=100, sequenceid=506, filesize=9.6 K 2024-11-15T11:53:46,649 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/B/fe0e98083562429ab524721be2e012b1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fe0e98083562429ab524721be2e012b1 2024-11-15T11:53:46,653 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fe0e98083562429ab524721be2e012b1, entries=100, sequenceid=506, filesize=9.6 K 2024-11-15T11:53:46,653 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/.tmp/C/55d73757537b4ac3b21fe4b7970595f2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/55d73757537b4ac3b21fe4b7970595f2 2024-11-15T11:53:46,656 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/55d73757537b4ac3b21fe4b7970595f2, entries=100, sequenceid=506, filesize=9.6 K 2024-11-15T11:53:46,657 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f47e749e6316d0ed90d8985c2e855dcb in 1264ms, sequenceid=506, compaction requested=true 2024-11-15T11:53:46,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/4110ac530ab44c498a1d226e87e41bae, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ae0cc90cbeac429aaaf98274d9570db1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/c8c633069e6244f695225d846045f6c9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8903ee165606489a9b5d3a6ca571312a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/9c8f27d7a5de47ad8c6d62d313e4018b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/2cd9611f911c4dd8adf5b94c2e23eb5b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d535c0a7c88846daad9c8dd2e06e3fff, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977] to archive 2024-11-15T11:53:46,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:46,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/725ee2717341452e9183109a8cd23d66 2024-11-15T11:53:46,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8027a02842ec4a6c8798a3072651c163 2024-11-15T11:53:46,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/04e801348c124712b56b64917a7ee8d6 2024-11-15T11:53:46,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/4110ac530ab44c498a1d226e87e41bae to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/4110ac530ab44c498a1d226e87e41bae 2024-11-15T11:53:46,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6163edbb0b254fd49720a138a7371ede 2024-11-15T11:53:46,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/23f0971f40be4771bf48ff2deeebae6c 2024-11-15T11:53:46,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ac1aadc35c3a4dc0b5584ed6caa488ba 2024-11-15T11:53:46,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ae0cc90cbeac429aaaf98274d9570db1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/ae0cc90cbeac429aaaf98274d9570db1 2024-11-15T11:53:46,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/3ce1b5a013594a34b126e9d819583cff 2024-11-15T11:53:46,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/aabfa3075aff40ae8fd3a3c0a2b2af4f 2024-11-15T11:53:46,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7ec703faeceb48419213fc6dff821c5f 2024-11-15T11:53:46,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/c8c633069e6244f695225d846045f6c9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/c8c633069e6244f695225d846045f6c9 2024-11-15T11:53:46,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cc9c03671cab4110880a047ccb601f2e 2024-11-15T11:53:46,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/53023d19b5694ef9861c1cb57428f5ca 2024-11-15T11:53:46,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8903ee165606489a9b5d3a6ca571312a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8903ee165606489a9b5d3a6ca571312a 2024-11-15T11:53:46,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/7e83515b30084b219979a345b65df9dd 2024-11-15T11:53:46,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/cf2adbf39a984775a07b137409f2f4ad 2024-11-15T11:53:46,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e2c127617fe34c24b38bae978add5d6d 2024-11-15T11:53:46,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/9c8f27d7a5de47ad8c6d62d313e4018b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/9c8f27d7a5de47ad8c6d62d313e4018b 2024-11-15T11:53:46,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/b4b8329bb56b4fd3a231551c1365c7e1 2024-11-15T11:53:46,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/2cd9611f911c4dd8adf5b94c2e23eb5b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/2cd9611f911c4dd8adf5b94c2e23eb5b 2024-11-15T11:53:46,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/f0f3d48c29734647b14c9cf6d2926cfe 2024-11-15T11:53:46,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e571937239e54d41be70e57d7c730d52 2024-11-15T11:53:46,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d0e8807faca344c2a3f5a134dc6b7b38 2024-11-15T11:53:46,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/6a625f0450884b3ea9f9af82630f693c 2024-11-15T11:53:46,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d535c0a7c88846daad9c8dd2e06e3fff to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d535c0a7c88846daad9c8dd2e06e3fff 2024-11-15T11:53:46,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/8972bde9062545e69c0e2c2d1434495e 2024-11-15T11:53:46,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/d951c78f50c544e7814709dbba883977 2024-11-15T11:53:46,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ed9e7af323e048929e8b6568e77c8894, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/0a61a548d1f34106a01d5cdc0cdfc942, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/465634307b9844fd80d79c8dfaf39b37, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/229aec5ddd034f78a087c376a903a685, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e4f0e8d157084ea58a94d892c8050354, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ba3b801d525f4579ad3f8bb4df3cf686, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/cbfdeab117204f928c9aad4a9636a670, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711] to archive 2024-11-15T11:53:46,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:46,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/2513df9e059445f7bf28b1d0c5aefb81 2024-11-15T11:53:46,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/adf8f154dbde4e07a2c6d403ad3047f4 2024-11-15T11:53:46,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ed9e7af323e048929e8b6568e77c8894 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ed9e7af323e048929e8b6568e77c8894 2024-11-15T11:53:46,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/05910922ce194612b1614ce56dbbbfe0 2024-11-15T11:53:46,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/809ac8bb2fff4523b6bd1e08f5b3dd1f 2024-11-15T11:53:46,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fb03a90dda7840b8a7a3312e170e666b 2024-11-15T11:53:46,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/0a61a548d1f34106a01d5cdc0cdfc942 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/0a61a548d1f34106a01d5cdc0cdfc942 2024-11-15T11:53:46,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a66b05fd1d6947d9abb19cd611d29a73 2024-11-15T11:53:46,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/a0e0cdb6844644f493a7085709575fb1 2024-11-15T11:53:46,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/b0278442bdb047bd87d5af09d62dc1a6 2024-11-15T11:53:46,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/465634307b9844fd80d79c8dfaf39b37 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/465634307b9844fd80d79c8dfaf39b37 2024-11-15T11:53:46,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/3edbd31f4544461cb971dde5ec6aa95f 2024-11-15T11:53:46,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/22e2cd663ef24aba9d74137823bc2fa3 2024-11-15T11:53:46,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/229aec5ddd034f78a087c376a903a685 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/229aec5ddd034f78a087c376a903a685 2024-11-15T11:53:46,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/775aca4db0334a38b3eaa08265654f26 2024-11-15T11:53:46,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/37e631779f9a44f2b4bdede6137fa6f9 2024-11-15T11:53:46,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/f702b085911149cd9eda9dcbc67145b0 2024-11-15T11:53:46,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e4f0e8d157084ea58a94d892c8050354 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e4f0e8d157084ea58a94d892c8050354 2024-11-15T11:53:46,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/9f612cc4f37743d6a10a29f3b4c9e3bb 2024-11-15T11:53:46,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/d5af95392d394e759d14cfef06a808c4 2024-11-15T11:53:46,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ba3b801d525f4579ad3f8bb4df3cf686 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ba3b801d525f4579ad3f8bb4df3cf686 2024-11-15T11:53:46,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/e42d35b572164382b42bff396867b83b 2024-11-15T11:53:46,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/c3fd4dab3ece4bdba0c671cc6c18e9df 2024-11-15T11:53:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/ae6db74f4bfc45f2b979cfe14fd80188 2024-11-15T11:53:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/cbfdeab117204f928c9aad4a9636a670 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/cbfdeab117204f928c9aad4a9636a670 2024-11-15T11:53:46,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/5b2ef90cc6954dca9b3ce55a00d12014 2024-11-15T11:53:46,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/029c2ddf220d44359b7185d1557ca62c 2024-11-15T11:53:46,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/77d403f0f5644e1b9a9fef4ed6394711 2024-11-15T11:53:46,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/49b4f2bef75f47f7983bf10f85edfb2d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef30ba4db6f44afb8f7f81d5b4b7b931, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ff39d63002e416591e44942d251ba6b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/be506b77658848f9a04f9b86d55a5d24, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fe756b72dfc345fe89843ec48b6e1654, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c024da49b64a496cafa111a214de0e4a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/1457270fa2424fe0a2f6351fd5995b10, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f] to archive 2024-11-15T11:53:46,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:53:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/50562059f9674be29bddc77804cc9e1c 2024-11-15T11:53:46,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a7854062a147444e80f7fcae3819f578 2024-11-15T11:53:46,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/49b4f2bef75f47f7983bf10f85edfb2d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/49b4f2bef75f47f7983bf10f85edfb2d 2024-11-15T11:53:46,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6ada7cbb3d4d4bf4999c2a8e71ab1198 2024-11-15T11:53:46,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/0950a0efbe4a4843af3036f75ceb23b2 2024-11-15T11:53:46,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ec5ba911fdd4f47bc729c7aaa76933b 2024-11-15T11:53:46,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef30ba4db6f44afb8f7f81d5b4b7b931 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef30ba4db6f44afb8f7f81d5b4b7b931 2024-11-15T11:53:46,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/efc925cbf3df48acb7c19c22126bf837 2024-11-15T11:53:46,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/31a0621930e243b98ad365d052f540a4 2024-11-15T11:53:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f98c96b0d3a0484c891fc770f7449f7b 2024-11-15T11:53:46,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ff39d63002e416591e44942d251ba6b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/7ff39d63002e416591e44942d251ba6b 2024-11-15T11:53:46,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/eda1117c4bcc416db9bbe6a2620b7d9b 2024-11-15T11:53:46,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5c091ed9ab364adb8d2a73bccf90f555 2024-11-15T11:53:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/be506b77658848f9a04f9b86d55a5d24 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/be506b77658848f9a04f9b86d55a5d24 2024-11-15T11:53:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c5ce7508464c4fd0ab57245544ec8eca 2024-11-15T11:53:46,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/b2ab5394e609442ca2c32e4714231469 2024-11-15T11:53:46,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fa478cc4577647a1a54cd4f1158c8747 2024-11-15T11:53:46,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fe756b72dfc345fe89843ec48b6e1654 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fe756b72dfc345fe89843ec48b6e1654 2024-11-15T11:53:46,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/f73d567244904d61b208b1f3ae3bedcd 2024-11-15T11:53:46,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ff373d1cfb5349e1bc99357bea26b168 2024-11-15T11:53:46,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c024da49b64a496cafa111a214de0e4a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/c024da49b64a496cafa111a214de0e4a 2024-11-15T11:53:46,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/a96da9492ea04a6abe64bc8a0e2bc9f4 2024-11-15T11:53:46,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/ef0ae66db94743d1a482008fb7b9db72 2024-11-15T11:53:46,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/3bb4385d1ac84124bba5629e0d4cadb9 2024-11-15T11:53:46,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/1457270fa2424fe0a2f6351fd5995b10 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/1457270fa2424fe0a2f6351fd5995b10 2024-11-15T11:53:46,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/d0a8ca890ade4f8ba9d25aeff0a72e59 2024-11-15T11:53:46,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/492faa5c41e8450e9ba80b187dd25408 2024-11-15T11:53:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/5f80b10f75c3401993a9284a5735863f 2024-11-15T11:53:46,736 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/recovered.edits/509.seqid, newMaxSeqId=509, maxSeqId=1 2024-11-15T11:53:46,736 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. 2024-11-15T11:53:46,736 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for f47e749e6316d0ed90d8985c2e855dcb: Waiting for close lock at 1731671625392Running coprocessor pre-close hooks at 1731671625392Disabling compacts and flushes for region at 1731671625392Disabling writes for close at 1731671625393 (+1 ms)Obtaining lock to block concurrent updates at 1731671625393Preparing flush snapshotting stores in f47e749e6316d0ed90d8985c2e855dcb at 1731671625393Finished memstore snapshotting TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731671625394 (+1 ms)Flushing stores of TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb. at 1731671625395 (+1 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/A: creating writer at 1731671625395Flushing f47e749e6316d0ed90d8985c2e855dcb/A: appending metadata at 1731671625400 (+5 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/A: closing flushed file at 1731671625400Flushing f47e749e6316d0ed90d8985c2e855dcb/B: creating writer at 1731671625810 (+410 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/B: appending metadata at 1731671625814 (+4 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/B: closing flushed file at 1731671625814Flushing f47e749e6316d0ed90d8985c2e855dcb/C: creating writer at 1731671626225 (+411 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/C: appending metadata at 1731671626230 (+5 ms)Flushing f47e749e6316d0ed90d8985c2e855dcb/C: closing flushed file at 1731671626230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fa75100: reopening flushed file at 1731671626640 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28379b56: reopening flushed file at 1731671626648 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b946aa: reopening flushed file at 1731671626653 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f47e749e6316d0ed90d8985c2e855dcb in 1264ms, sequenceid=506, compaction requested=true at 1731671626657 (+4 ms)Writing region close event to WAL at 1731671626733 (+76 ms)Running coprocessor post-close hooks at 1731671626736 (+3 ms)Closed at 1731671626736 2024-11-15T11:53:46,738 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:46,739 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f47e749e6316d0ed90d8985c2e855dcb, regionState=CLOSED 2024-11-15T11:53:46,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:46,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T11:53:46,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure f47e749e6316d0ed90d8985c2e855dcb, server=7adf9b3d9d04,44479,1731671471201 in 1.5010 sec 2024-11-15T11:53:46,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T11:53:46,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f47e749e6316d0ed90d8985c2e855dcb, UNASSIGN in 1.5070 sec 2024-11-15T11:53:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T11:53:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-15T11:53:46,746 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671626746"}]},"ts":"1731671626746"} 2024-11-15T11:53:46,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:53:46,747 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:53:46,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-15T11:53:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T11:53:47,382 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:47,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:53:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,387 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T11:53:47,389 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,392 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:47,395 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/recovered.edits] 2024-11-15T11:53:47,419 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e71bd08b96464890bb73458135d77779 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/e71bd08b96464890bb73458135d77779 2024-11-15T11:53:47,420 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/eed8995d28204b01a3959e53ac218411 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/eed8995d28204b01a3959e53ac218411 2024-11-15T11:53:47,421 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/fe87cb89bab443da95bf900ee67b17cf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/A/fe87cb89bab443da95bf900ee67b17cf 2024-11-15T11:53:47,422 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/68bd93fb3d894986a82967d15e58859c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/68bd93fb3d894986a82967d15e58859c 2024-11-15T11:53:47,423 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/82594d58562a4aada2cf1dae4b7b1ede to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/82594d58562a4aada2cf1dae4b7b1ede 2024-11-15T11:53:47,424 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fe0e98083562429ab524721be2e012b1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/B/fe0e98083562429ab524721be2e012b1 2024-11-15T11:53:47,426 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/55d73757537b4ac3b21fe4b7970595f2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/55d73757537b4ac3b21fe4b7970595f2 2024-11-15T11:53:47,427 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6854aac641a0426c99b241bc24da9e99 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/6854aac641a0426c99b241bc24da9e99 2024-11-15T11:53:47,428 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fb0463c523de43f6a92d03c708d18803 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/C/fb0463c523de43f6a92d03c708d18803 2024-11-15T11:53:47,429 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/recovered.edits/509.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb/recovered.edits/509.seqid 2024-11-15T11:53:47,430 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/f47e749e6316d0ed90d8985c2e855dcb 2024-11-15T11:53:47,430 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:53:47,431 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,433 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:53:47,434 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:53:47,435 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,435 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:53:47,435 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671627435"}]},"ts":"9223372036854775807"} 2024-11-15T11:53:47,437 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:53:47,437 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f47e749e6316d0ed90d8985c2e855dcb, NAME => 'TestAcidGuarantees,,1731671597133.f47e749e6316d0ed90d8985c2e855dcb.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:53:47,437 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:53:47,437 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671627437"}]},"ts":"9223372036854775807"} 2024-11-15T11:53:47,438 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:53:47,439 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-15T11:53:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T11:53:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:53:47,493 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:47,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 250), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 529), ProcessCount=11 (was 11), AvailableMemoryMB=10832 (was 11629) 2024-11-15T11:53:47,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=10832 2024-11-15T11:53:47,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:53:47,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:53:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:47,516 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T11:53:47,516 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:47,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-15T11:53:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:47,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T11:53:47,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742378_1554 (size=1061) 2024-11-15T11:53:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:47,927 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186 2024-11-15T11:53:47,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742379_1555 (size=53) 2024-11-15T11:53:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:48,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:53:48,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 698b0d18c06b908fd10b74c4eb30c7d2, disabling compactions & flushes 2024-11-15T11:53:48,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. after waiting 0 ms 2024-11-15T11:53:48,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 698b0d18c06b908fd10b74c4eb30c7d2: Waiting for close lock at 1731671628337Disabling compacts and flushes for region at 1731671628337Disabling writes for close at 1731671628337Writing region close event to WAL at 1731671628337Closed at 1731671628337 2024-11-15T11:53:48,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T11:53:48,339 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731671628339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731671628339"}]},"ts":"1731671628339"} 2024-11-15T11:53:48,341 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T11:53:48,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T11:53:48,343 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671628343"}]},"ts":"1731671628343"} 2024-11-15T11:53:48,345 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T11:53:48,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, ASSIGN}] 2024-11-15T11:53:48,346 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, ASSIGN 2024-11-15T11:53:48,347 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, ASSIGN; state=OFFLINE, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=false 2024-11-15T11:53:48,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:48,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, ASSIGN because future has completed 2024-11-15T11:53:48,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:48,659 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,660 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:53:48,660 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,660 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:53:48,660 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,660 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,661 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,662 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:48,663 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName A 2024-11-15T11:53:48,663 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:48,663 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:48,663 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,664 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:48,664 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName B 2024-11-15T11:53:48,664 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:48,665 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:48,665 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,666 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:48,666 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName C 2024-11-15T11:53:48,666 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:48,666 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:48,667 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,667 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,667 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,668 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,668 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,668 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,669 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:53:48,670 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,672 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T11:53:48,672 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 698b0d18c06b908fd10b74c4eb30c7d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75260956, jitterRate=0.12147563695907593}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:53:48,672 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:48,673 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 698b0d18c06b908fd10b74c4eb30c7d2: Running coprocessor pre-open hook at 1731671628660Writing region info on filesystem at 1731671628660Initializing all the Stores at 1731671628661 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671628661Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671628661Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671628661Cleaning up temporary data from old regions at 1731671628668 (+7 ms)Running coprocessor post-open hooks at 1731671628672 (+4 ms)Region opened successfully at 1731671628673 (+1 ms) 2024-11-15T11:53:48,674 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., pid=146, masterSystemTime=1731671628657 2024-11-15T11:53:48,676 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,676 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:48,676 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=OPEN, openSeqNum=2, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:48,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:48,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T11:53:48,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 in 176 msec 2024-11-15T11:53:48,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T11:53:48,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, ASSIGN in 334 msec 2024-11-15T11:53:48,681 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T11:53:48,681 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671628681"}]},"ts":"1731671628681"} 2024-11-15T11:53:48,682 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T11:53:48,683 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T11:53:48,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-15T11:53:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T11:53:49,662 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:49,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:49,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:49,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:49,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:49,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:49,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:49,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307099d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:49,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:49,669 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9d076b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:49,670 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:49,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:49,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:49,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:49,674 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T11:53:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7adf9b3d9d04,36957,1731671470483 2024-11-15T11:53:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77c62ea3 2024-11-15T11:53:49,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T11:53:49,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T11:53:49,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T11:53:49,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T11:53:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T11:53:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742380_1556 (size=1097) 2024-11-15T11:53:50,093 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-15T11:53:50,093 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-15T11:53:50,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:53:50,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE}] 2024-11-15T11:53:50,104 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE 2024-11-15T11:53:50,106 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE because future has completed 2024-11-15T11:53:50,107 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:53:50,107 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:50,260 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,260 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:53:50,260 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 698b0d18c06b908fd10b74c4eb30c7d2, disabling compactions & flushes 2024-11-15T11:53:50,260 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,260 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,260 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. after waiting 0 ms 2024-11-15T11:53:50,260 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,267 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T11:53:50,269 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,269 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 698b0d18c06b908fd10b74c4eb30c7d2: Waiting for close lock at 1731671630260Running coprocessor pre-close hooks at 1731671630260Disabling compacts and flushes for region at 1731671630260Disabling writes for close at 1731671630260Writing region close event to WAL at 1731671630261 (+1 ms)Running coprocessor post-close hooks at 1731671630268 (+7 ms)Closed at 1731671630268 2024-11-15T11:53:50,269 WARN [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 698b0d18c06b908fd10b74c4eb30c7d2 to self. 2024-11-15T11:53:50,271 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,272 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=CLOSED 2024-11-15T11:53:50,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:50,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-15T11:53:50,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 in 167 msec 2024-11-15T11:53:50,276 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE; state=CLOSED, location=7adf9b3d9d04,44479,1731671471201; forceNewPlan=false, retain=true 2024-11-15T11:53:50,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=OPENING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE because future has completed 2024-11-15T11:53:50,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:53:50,590 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,590 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} 2024-11-15T11:53:50,590 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,590 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T11:53:50,590 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,591 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,592 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,592 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:50,593 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName A 2024-11-15T11:53:50,594 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:50,594 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:50,595 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,595 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:50,595 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName B 2024-11-15T11:53:50,595 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:50,596 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:50,596 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,596 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-15T11:53:50,597 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698b0d18c06b908fd10b74c4eb30c7d2 columnFamilyName C 2024-11-15T11:53:50,597 DEBUG [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:50,597 INFO [StoreOpener-698b0d18c06b908fd10b74c4eb30c7d2-1 {}] regionserver.HStore(327): Store=698b0d18c06b908fd10b74c4eb30c7d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T11:53:50,597 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,598 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,598 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,599 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,600 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T11:53:50,602 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,602 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 698b0d18c06b908fd10b74c4eb30c7d2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72867204, jitterRate=0.08580595254898071}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T11:53:50,602 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,603 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 698b0d18c06b908fd10b74c4eb30c7d2: Running coprocessor pre-open hook at 1731671630591Writing region info on filesystem at 1731671630591Initializing all the Stores at 1731671630591Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671630591Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671630592 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731671630592Cleaning up temporary data from old regions at 1731671630600 (+8 ms)Running coprocessor post-open hooks at 1731671630602 (+2 ms)Region opened successfully at 1731671630603 (+1 ms) 2024-11-15T11:53:50,604 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., pid=151, masterSystemTime=1731671630587 2024-11-15T11:53:50,605 DEBUG [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,605 INFO [RS_OPEN_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=OPEN, openSeqNum=5, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:53:50,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-15T11:53:50,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 in 176 msec 2024-11-15T11:53:50,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T11:53:50,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, REOPEN/MOVE in 507 msec 2024-11-15T11:53:50,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T11:53:50,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-15T11:53:50,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-15T11:53:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T11:53:50,614 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:50,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e70e8a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,617 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9128fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ecc483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,619 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70698795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,621 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325974c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,622 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4e52c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1e8592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4183aff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,627 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa5e9a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c7b0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c166e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547f8def, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cce6e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,633 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313a21a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,635 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e2a1d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,636 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760d45f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,637 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,638 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44da02cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b73da4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7adf9b3d9d04,36957,-1 for getting cluster id 2024-11-15T11:53:50,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T11:53:50,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '52397efa-b1b0-4774-8091-97e7ccb77a0b' 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "52397efa-b1b0-4774-8091-97e7ccb77a0b" 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce4b2dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7adf9b3d9d04,36957,-1] 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T11:53:50,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:53:50,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T11:53:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e5b99f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T11:53:50,643 DEBUG [Thread-2482 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,643 DEBUG [Thread-2484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,643 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,643 DEBUG [Thread-2486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,643 DEBUG [Thread-2467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,644 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:50,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,645 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T11:53:50,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,647 DEBUG [Thread-2478 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:50,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,648 DEBUG [Thread-2480 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,648 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:50,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:50,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:50,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:50,663 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T11:53:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7adf9b3d9d04,44479,1731671471201, seqNum=-1] 2024-11-15T11:53:50,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T11:53:50,667 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T11:53:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5] 2024-11-15T11:53:50,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151bea96b908a5419986ad604608a7bb6b_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671630654/Put/seqid=0 2024-11-15T11:53:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742381_1557 (size=14594) 2024-11-15T11:53:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671690683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671690686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731671630680, max timestamp=1731671630688Num uniques 859; 2024-11-15T11:53:50,690 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:53:50,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731671630680, max timestamp=1731671630688Num uniques 859; 2024-11-15T11:53:50,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:53:50,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731671630680, max timestamp=1731671630688Num uniques 859; 2024-11-15T11:53:50,691 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-15T11:53:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671690709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671690710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671690711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:50,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671690800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671690800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671690821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671690821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671690821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:50,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:50,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:51,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671691010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671691010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671691031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671691031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671691031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,085 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:51,088 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151bea96b908a5419986ad604608a7bb6b_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bea96b908a5419986ad604608a7bb6b_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:51,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3554f1671e1a4da2b3e8d7332774524a, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:51,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3554f1671e1a4da2b3e8d7332774524a is 175, key is test_row_0/A:col10/1731671630654/Put/seqid=0 2024-11-15T11:53:51,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742382_1558 (size=39549) 2024-11-15T11:53:51,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3554f1671e1a4da2b3e8d7332774524a 2024-11-15T11:53:51,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/7ad234c42dee4e759ad0f2394d8e40e6 is 50, key is test_row_0/B:col10/1731671630654/Put/seqid=0 2024-11-15T11:53:51,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742383_1559 (size=12001) 2024-11-15T11:53:51,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:51,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671691320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671691320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671691340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671691341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671691341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/7ad234c42dee4e759ad0f2394d8e40e6 2024-11-15T11:53:51,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/818db91827ea4001a06194a5a54234a9 is 50, key is test_row_0/C:col10/1731671630654/Put/seqid=0 2024-11-15T11:53:51,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742384_1560 (size=12001) 2024-11-15T11:53:51,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671691831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671691831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671691850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671691850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671691850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:51,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:51,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:51,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/818db91827ea4001a06194a5a54234a9 2024-11-15T11:53:51,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3554f1671e1a4da2b3e8d7332774524a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a 2024-11-15T11:53:51,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a, entries=200, sequenceid=15, filesize=38.6 K 2024-11-15T11:53:51,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/7ad234c42dee4e759ad0f2394d8e40e6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6 2024-11-15T11:53:51,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T11:53:51,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/818db91827ea4001a06194a5a54234a9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9 2024-11-15T11:53:51,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T11:53:51,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1294ms, sequenceid=15, compaction requested=false 2024-11-15T11:53:51,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:52,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:52,023 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:52,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154bf25312ae65455f963bf36e321f1def_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671630710/Put/seqid=0 2024-11-15T11:53:52,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742385_1561 (size=12154) 2024-11-15T11:53:52,260 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T11:53:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:52,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154bf25312ae65455f963bf36e321f1def_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154bf25312ae65455f963bf36e321f1def_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/8521e4012d024a15be62e9e86dd9672f, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/8521e4012d024a15be62e9e86dd9672f is 175, key is test_row_0/A:col10/1731671630710/Put/seqid=0 2024-11-15T11:53:52,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742386_1562 (size=30955) 2024-11-15T11:53:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:52,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:52,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671692859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671692861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671692861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,864 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:52,864 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671692863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671692870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,873 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_2' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/8521e4012d024a15be62e9e86dd9672f 2024-11-15T11:53:52,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/4b546faf3d714f8db24cdb21549739f1 is 50, key is test_row_0/B:col10/1731671630710/Put/seqid=0 2024-11-15T11:53:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742387_1563 (size=12001) 2024-11-15T11:53:52,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671692970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671692970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:53,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671693181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:53,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671693181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:53,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/4b546faf3d714f8db24cdb21549739f1 2024-11-15T11:53:53,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/345e5674b5934e9ea30549613a086429 is 50, key is test_row_0/C:col10/1731671630710/Put/seqid=0 2024-11-15T11:53:53,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742388_1564 (size=12001) 2024-11-15T11:53:53,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671693490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:53,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671693490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:53,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:53,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:53,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/345e5674b5934e9ea30549613a086429 2024-11-15T11:53:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/8521e4012d024a15be62e9e86dd9672f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f 2024-11-15T11:53:53,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f, entries=150, sequenceid=49, filesize=30.2 K 2024-11-15T11:53:53,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/4b546faf3d714f8db24cdb21549739f1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1 2024-11-15T11:53:53,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T11:53:53,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/345e5674b5934e9ea30549613a086429 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429 2024-11-15T11:53:53,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429, entries=150, sequenceid=49, filesize=11.7 K 2024-11-15T11:53:53,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1683ms, sequenceid=49, compaction requested=false 2024-11-15T11:53:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T11:53:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T11:53:53,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T11:53:53,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0590 sec 2024-11-15T11:53:53,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.0640 sec 2024-11-15T11:53:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:54,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:54,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:54,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ce2e260cfac4417987f7e1777b9b9ddb_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:54,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742389_1565 (size=14594) 2024-11-15T11:53:54,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671694044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671694044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671694160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671694160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671694370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671694370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,412 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:54,414 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ce2e260cfac4417987f7e1777b9b9ddb_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ce2e260cfac4417987f7e1777b9b9ddb_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:54,415 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/5ca9b2c007984e5780636cef3afbd8ea, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/5ca9b2c007984e5780636cef3afbd8ea is 175, key is test_row_0/A:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:54,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742390_1566 (size=39549) 2024-11-15T11:53:54,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/5ca9b2c007984e5780636cef3afbd8ea 2024-11-15T11:53:54,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/72dde905da984fbe88be7869f33cb324 is 50, key is test_row_0/B:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:54,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742391_1567 (size=12001) 2024-11-15T11:53:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671694681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671694681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T11:53:54,800 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:54,800 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T11:53:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:54,802 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:54,802 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:54,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:54,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/72dde905da984fbe88be7869f33cb324 2024-11-15T11:53:54,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/352809db8cfb42c986ae146ce619f265 is 50, key is test_row_0/C:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:54,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742392_1568 (size=12001) 2024-11-15T11:53:54,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/352809db8cfb42c986ae146ce619f265 2024-11-15T11:53:54,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/5ca9b2c007984e5780636cef3afbd8ea as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea 2024-11-15T11:53:54,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea, entries=200, sequenceid=62, filesize=38.6 K 2024-11-15T11:53:54,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/72dde905da984fbe88be7869f33cb324 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324 2024-11-15T11:53:54,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T11:53:54,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/352809db8cfb42c986ae146ce619f265 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265 2024-11-15T11:53:54,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265, entries=150, sequenceid=62, filesize=11.7 K 2024-11-15T11:53:54,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 698b0d18c06b908fd10b74c4eb30c7d2 in 850ms, sequenceid=62, compaction requested=true 2024-11-15T11:53:54,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:54,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:54,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:54,852 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:54,853 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:54,853 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:54,853 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:54,853 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:53:54,853 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:53:54,853 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:54,853 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:54,853 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=35.2 K 2024-11-15T11:53:54,853 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=107.5 K 2024-11-15T11:53:54,853 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:54,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea] 2024-11-15T11:53:54,854 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ad234c42dee4e759ad0f2394d8e40e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671630652 2024-11-15T11:53:54,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3554f1671e1a4da2b3e8d7332774524a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671630650 2024-11-15T11:53:54,854 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b546faf3d714f8db24cdb21549739f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671630699 2024-11-15T11:53:54,854 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8521e4012d024a15be62e9e86dd9672f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671630699 2024-11-15T11:53:54,857 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ca9b2c007984e5780636cef3afbd8ea, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:54,857 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 72dde905da984fbe88be7869f33cb324, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:54,879 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,880 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:54,880 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/18484b6c9a094d82a37d09ec2fa458a5 is 50, key is test_row_0/B:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:54,881 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115968207481a13430292d1f347f4879cfd_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115968207481a13430292d1f347f4879cfd_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,883 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115968207481a13430292d1f347f4879cfd_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:54,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634037, max timestamp=1731671634883Num uniques 859; 2024-11-15T11:53:54,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634037, max timestamp=1731671634883Num uniques 859; 2024-11-15T11:53:54,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634037, max timestamp=1731671634883Num uniques 859; 2024-11-15T11:53:54,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:54,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:54,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742393_1569 (size=12104) 2024-11-15T11:53:54,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111551063a1dc5cf474fa5eb49831f510bac_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671634042/Put/seqid=0 2024-11-15T11:53:54,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742394_1570 (size=4469) 2024-11-15T11:53:54,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742395_1571 (size=12154) 2024-11-15T11:53:54,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:54,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111551063a1dc5cf474fa5eb49831f510bac_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111551063a1dc5cf474fa5eb49831f510bac_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:54,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/336d81ffb3e94a12bc6a29a488bfc833, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:54,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/336d81ffb3e94a12bc6a29a488bfc833 is 175, key is test_row_0/A:col10/1731671634042/Put/seqid=0 2024-11-15T11:53:54,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742396_1572 (size=30955) 2024-11-15T11:53:54,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/336d81ffb3e94a12bc6a29a488bfc833 2024-11-15T11:53:54,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/81ed03277cf84d0d8fdf57fbb2779832 is 50, key is test_row_0/B:col10/1731671634042/Put/seqid=0 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634906, max timestamp=1731671634920Num uniques 859; 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634906, max timestamp=1731671634920Num uniques 859; 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731671634906, max timestamp=1731671634920Num uniques 859; 2024-11-15T11:53:54,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-15T11:53:54,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742397_1573 (size=12001) 2024-11-15T11:53:54,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:54,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:54,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:54,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:54,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671634949, max timestamp=1731671634957Num uniques 1018; 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671634949, max timestamp=1731671634957Num uniques 1018; 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731671634949, max timestamp=1731671634957Num uniques 1018; 2024-11-15T11:53:54,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-15T11:53:54,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671634971, max timestamp=1731671634992Num uniques 1077; 2024-11-15T11:53:54,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T11:53:54,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671634971, max timestamp=1731671634992Num uniques 1077; 2024-11-15T11:53:54,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T11:53:54,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731671634971, max timestamp=1731671634992Num uniques 1077; 2024-11-15T11:53:54,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-15T11:53:55,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731671635016, max timestamp=1731671635030Num uniques 1086; 2024-11-15T11:53:55,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-15T11:53:55,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731671635016, max timestamp=1731671635030Num uniques 1086; 2024-11-15T11:53:55,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-15T11:53:55,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731671635016, max timestamp=1731671635030Num uniques 1086; 2024-11-15T11:53:55,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-15T11:53:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671695063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731671635059, max timestamp=1731671635068Num uniques 1145; 2024-11-15T11:53:55,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8928, totalHeapSize=20520, min timestamp=1731671635059, max timestamp=1731671635068Num uniques 1145; 2024-11-15T11:53:55,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-15T11:53:55,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52438, new segment data size=8928, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-15T11:53:55,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8920, totalHeapSize=20520, min timestamp=1731671635059, max timestamp=1731671635068Num uniques 1145; 2024-11-15T11:53:55,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52430, new segment data size=8920, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-15T11:53:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731671635112, max timestamp=1731671635130Num uniques 1054; 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731671635112, max timestamp=1731671635130Num uniques 1054; 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731671635112, max timestamp=1731671635130Num uniques 1054; 2024-11-15T11:53:55,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731671635125, max timestamp=1731671635188Num uniques 1063; 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731671635125, max timestamp=1731671635188Num uniques 1063; 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731671635125, max timestamp=1731671635188Num uniques 1063; 2024-11-15T11:53:55,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-15T11:53:55,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731671635211, max timestamp=1731671635221Num uniques 1072; 2024-11-15T11:53:55,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731671635211, max timestamp=1731671635221Num uniques 1072; 2024-11-15T11:53:55,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-15T11:53:55,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-15T11:53:55,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=372, cellSize=17032, totalHeapSize=38928, min timestamp=1731671635200, max timestamp=1731671635221Num uniques 1072; 2024-11-15T11:53:55,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=17032, suffix heap size=139456,new segment heap size=38928 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=372 2024-11-15T11:53:55,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671695249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671695250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671695250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671695252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731671635242, max timestamp=1731671635256Num uniques 1081; 2024-11-15T11:53:55,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731671635242, max timestamp=1731671635256Num uniques 1231; 2024-11-15T11:53:55,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=56376, new segment data size=8286, suffix heap size=155992,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1231, new segment cells count=181 2024-11-15T11:53:55,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-15T11:53:55,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731671635242, max timestamp=1731671635256Num uniques 1081; 2024-11-15T11:53:55,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,296 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/18484b6c9a094d82a37d09ec2fa458a5 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/18484b6c9a094d82a37d09ec2fa458a5 2024-11-15T11:53:55,299 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 18484b6c9a094d82a37d09ec2fa458a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:55,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:55,299 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671634852; duration=0sec 2024-11-15T11:53:55,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:55,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:53:55,299 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:55,299 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:55,300 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/86a2f09362cf4b37861cf3353b009063 is 175, key is test_row_0/A:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:55,301 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:55,301 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:53:55,301 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,301 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=35.2 K 2024-11-15T11:53:55,301 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 818db91827ea4001a06194a5a54234a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731671630652 2024-11-15T11:53:55,302 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 345e5674b5934e9ea30549613a086429, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731671630699 2024-11-15T11:53:55,302 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 352809db8cfb42c986ae146ce619f265, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:55,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742398_1574 (size=31058) 2024-11-15T11:53:55,308 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:55,309 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/92f4f9a3fed34b62ade5d1618f3bb5d4 is 50, key is test_row_0/C:col10/1731671632858/Put/seqid=0 2024-11-15T11:53:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742399_1575 (size=12104) 2024-11-15T11:53:55,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/81ed03277cf84d0d8fdf57fbb2779832 2024-11-15T11:53:55,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/35484727425b4623927b53df1254ff7a is 50, key is test_row_0/C:col10/1731671634042/Put/seqid=0 2024-11-15T11:53:55,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742400_1576 (size=12001) 2024-11-15T11:53:55,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/35484727425b4623927b53df1254ff7a 2024-11-15T11:53:55,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/336d81ffb3e94a12bc6a29a488bfc833 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833 2024-11-15T11:53:55,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833, entries=150, sequenceid=84, filesize=30.2 K 2024-11-15T11:53:55,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/81ed03277cf84d0d8fdf57fbb2779832 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832 2024-11-15T11:53:55,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T11:53:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/35484727425b4623927b53df1254ff7a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a 2024-11-15T11:53:55,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731671635318, max timestamp=1731671635342Num uniques 1040; 2024-11-15T11:53:55,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731671635318, max timestamp=1731671635342Num uniques 1040; 2024-11-15T11:53:55,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T11:53:55,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T11:53:55,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731671635318, max timestamp=1731671635342Num uniques 1040; 2024-11-15T11:53:55,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-15T11:53:55,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a, entries=150, sequenceid=84, filesize=11.7 K 2024-11-15T11:53:55,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=26.84 KB/27480 for 698b0d18c06b908fd10b74c4eb30c7d2 in 462ms, sequenceid=84, compaction requested=false 2024-11-15T11:53:55,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:55,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=53.67 KB heapSize=143.02 KB 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:55,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115787045640b5b45da8750f68ed61a53f0_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:55,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742401_1577 (size=14994) 2024-11-15T11:53:55,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671635382, max timestamp=1731671635393Num uniques 859; 2024-11-15T11:53:55,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:55,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671635382, max timestamp=1731671635393Num uniques 859; 2024-11-15T11:53:55,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731671635382, max timestamp=1731671635393Num uniques 859; 2024-11-15T11:53:55,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:55,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-15T11:53:55,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671695413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671695414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671695415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671695416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671695420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671695530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671695530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671695530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671695530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671695530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,707 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/86a2f09362cf4b37861cf3353b009063 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063 2024-11-15T11:53:55,710 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 86a2f09362cf4b37861cf3353b009063(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:55,711 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:55,711 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671634852; duration=0sec 2024-11-15T11:53:55,711 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:55,711 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:53:55,714 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/92f4f9a3fed34b62ade5d1618f3bb5d4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/92f4f9a3fed34b62ade5d1618f3bb5d4 2024-11-15T11:53:55,717 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into 92f4f9a3fed34b62ade5d1618f3bb5d4(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:55,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:55,717 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671634853; duration=0sec 2024-11-15T11:53:55,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:55,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:53:55,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671695740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671695740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671695740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671695740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671695740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:55,767 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:55,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115787045640b5b45da8750f68ed61a53f0_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115787045640b5b45da8750f68ed61a53f0_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:55,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d2ad5dc5887a4491b9e5692292012c3d, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:55,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d2ad5dc5887a4491b9e5692292012c3d is 175, key is test_row_0/A:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:55,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742402_1578 (size=39949) 2024-11-15T11:53:55,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:55,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:55,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:55,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:56,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671696050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671696050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671696050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671696050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671696050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d2ad5dc5887a4491b9e5692292012c3d 2024-11-15T11:53:56,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/8efde97143424f7db3e34b7a72b7a6bf is 50, key is test_row_0/B:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:56,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742403_1579 (size=12301) 2024-11-15T11:53:56,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671696560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671696560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671696560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671696560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671696560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:56,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:56,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/8efde97143424f7db3e34b7a72b7a6bf 2024-11-15T11:53:56,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/568311e47f8d4dae9b22010d2663b02f is 50, key is test_row_0/C:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:56,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742404_1580 (size=12301) 2024-11-15T11:53:56,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:56,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:56,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/568311e47f8d4dae9b22010d2663b02f 2024-11-15T11:53:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d2ad5dc5887a4491b9e5692292012c3d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d 2024-11-15T11:53:56,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d, entries=200, sequenceid=264, filesize=39.0 K 2024-11-15T11:53:57,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/8efde97143424f7db3e34b7a72b7a6bf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf 2024-11-15T11:53:57,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T11:53:57,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/568311e47f8d4dae9b22010d2663b02f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f 2024-11-15T11:53:57,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f, entries=150, sequenceid=264, filesize=12.0 K 2024-11-15T11:53:57,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~142.97 KB/146400, currentSize=134.18 KB/137400 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1654ms, sequenceid=264, compaction requested=true 2024-11-15T11:53:57,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:57,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101962 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:53:57,014 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:53:57,014 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,014 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,015 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=99.6 K 2024-11-15T11:53:57,015 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/18484b6c9a094d82a37d09ec2fa458a5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=35.6 K 2024-11-15T11:53:57,015 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d] 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 18484b6c9a094d82a37d09ec2fa458a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a2f09362cf4b37861cf3353b009063, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ed03277cf84d0d8fdf57fbb2779832, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671634037 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336d81ffb3e94a12bc6a29a488bfc833, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671634037 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8efde97143424f7db3e34b7a72b7a6bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635350 2024-11-15T11:53:57,015 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ad5dc5887a4491b9e5692292012c3d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635330 2024-11-15T11:53:57,019 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:57,021 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:57,021 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/30b7087c90164c9191b381a7f0f0326d is 50, key is test_row_0/B:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:57,028 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a4fe1a72237f431fb2076f5fa708d8da_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:57,030 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a4fe1a72237f431fb2076f5fa708d8da_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:57,031 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a4fe1a72237f431fb2076f5fa708d8da_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:57,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742405_1581 (size=12507) 2024-11-15T11:53:57,039 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/30b7087c90164c9191b381a7f0f0326d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/30b7087c90164c9191b381a7f0f0326d 2024-11-15T11:53:57,042 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 30b7087c90164c9191b381a7f0f0326d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:57,042 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:57,042 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671637014; duration=0sec 2024-11-15T11:53:57,042 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:57,042 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:53:57,042 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:57,043 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:57,043 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:53:57,043 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,043 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/92f4f9a3fed34b62ade5d1618f3bb5d4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=35.6 K 2024-11-15T11:53:57,044 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f4f9a3fed34b62ade5d1618f3bb5d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731671632858 2024-11-15T11:53:57,044 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 35484727425b4623927b53df1254ff7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731671634037 2024-11-15T11:53:57,044 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 568311e47f8d4dae9b22010d2663b02f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635350 2024-11-15T11:53:57,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742406_1582 (size=4469) 2024-11-15T11:53:57,049 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:57,050 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/39c63d6a78ec44ce810b47737caf33bb is 50, key is test_row_0/C:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742407_1583 (size=12507) 2024-11-15T11:53:57,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:57,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115495a65a65fcf46bb917316f6476dd722_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671635419/Put/seqid=0 2024-11-15T11:53:57,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742408_1584 (size=12454) 2024-11-15T11:53:57,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:57,105 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115495a65a65fcf46bb917316f6476dd722_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115495a65a65fcf46bb917316f6476dd722_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d09f4b47979047148d14d6f8b1fcc5c4, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:57,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d09f4b47979047148d14d6f8b1fcc5c4 is 175, key is test_row_0/A:col10/1731671635419/Put/seqid=0 2024-11-15T11:53:57,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742409_1585 (size=31255) 2024-11-15T11:53:57,112 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d09f4b47979047148d14d6f8b1fcc5c4 2024-11-15T11:53:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/dbf2640921f040f681457eb1f3a64dba is 50, key is test_row_0/B:col10/1731671635419/Put/seqid=0 2024-11-15T11:53:57,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742410_1586 (size=12301) 2024-11-15T11:53:57,449 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:57,449 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68fe000702274a03b4930fe5267d9ce0 is 175, key is test_row_0/A:col10/1731671635355/Put/seqid=0 2024-11-15T11:53:57,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742411_1587 (size=31461) 2024-11-15T11:53:57,496 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/39c63d6a78ec44ce810b47737caf33bb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/39c63d6a78ec44ce810b47737caf33bb 2024-11-15T11:53:57,499 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into 39c63d6a78ec44ce810b47737caf33bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:57,499 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:57,499 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671637014; duration=0sec 2024-11-15T11:53:57,499 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:57,499 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:53:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/dbf2640921f040f681457eb1f3a64dba 2024-11-15T11:53:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8c86a84b559d4320ba568232ae03702a is 50, key is test_row_0/C:col10/1731671635419/Put/seqid=0 2024-11-15T11:53:57,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742412_1588 (size=12301) 2024-11-15T11:53:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:57,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:57,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671697580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671697580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,585 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:57,585 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671697583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671697584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671697585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671697690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671697700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671697700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,897 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68fe000702274a03b4930fe5267d9ce0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0 2024-11-15T11:53:57,901 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 68fe000702274a03b4930fe5267d9ce0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:57,901 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:57,901 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671637014; duration=0sec 2024-11-15T11:53:57,901 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:57,901 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:53:57,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671697900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671697910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671697910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:57,942 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8c86a84b559d4320ba568232ae03702a 2024-11-15T11:53:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/d09f4b47979047148d14d6f8b1fcc5c4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4 2024-11-15T11:53:57,948 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4, entries=150, sequenceid=302, filesize=30.5 K 2024-11-15T11:53:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/dbf2640921f040f681457eb1f3a64dba as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba 2024-11-15T11:53:57,952 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T11:53:57,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8c86a84b559d4320ba568232ae03702a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a 2024-11-15T11:53:57,956 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a, entries=150, sequenceid=302, filesize=12.0 K 2024-11-15T11:53:57,957 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 698b0d18c06b908fd10b74c4eb30c7d2 in 864ms, sequenceid=302, compaction requested=false 2024-11-15T11:53:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:57,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T11:53:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T11:53:57,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T11:53:57,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1560 sec 2024-11-15T11:53:57,961 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.1600 sec 2024-11-15T11:53:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:58,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:58,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:53:58,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111509b25baed5cd407786a8f10c23b26b02_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:58,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742413_1589 (size=14994) 2024-11-15T11:53:58,249 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671638233, max timestamp=1731671638247Num uniques 859; 2024-11-15T11:53:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671638233, max timestamp=1731671638247Num uniques 859; 2024-11-15T11:53:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731671638233, max timestamp=1731671638247Num uniques 859; 2024-11-15T11:53:58,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7adf9b3d9d04:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-15T11:53:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671698280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671698281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671698282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671698390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671698390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671698390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671698601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671698601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671698601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:58,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111509b25baed5cd407786a8f10c23b26b02_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111509b25baed5cd407786a8f10c23b26b02_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:58,625 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1468679297294027977aec328760b4fe, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:58,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1468679297294027977aec328760b4fe is 175, key is test_row_0/A:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:58,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742414_1590 (size=39949) 2024-11-15T11:53:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671698910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671698910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671698910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:58,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T11:53:58,961 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:53:58,961 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:53:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T11:53:58,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:53:58,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:53:58,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:53:58,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:53:59,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1468679297294027977aec328760b4fe 2024-11-15T11:53:59,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/560c1d62e3dc48d09b452b65ea58ee4c is 50, key is test_row_0/B:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:59,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742415_1591 (size=12301) 2024-11-15T11:53:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:53:59,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:59,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:53:59,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671699420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671699420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671699420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:59,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/560c1d62e3dc48d09b452b65ea58ee4c 2024-11-15T11:53:59,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/e6820e91dd2945d392d42473040bbd7f is 50, key is test_row_0/C:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:59,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742416_1592 (size=12301) 2024-11-15T11:53:59,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:53:59,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671699590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:59,593 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:59,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:53:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671699600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:53:59,604 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:53:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:53:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:53:59,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:53:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:53:59,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/e6820e91dd2945d392d42473040bbd7f 2024-11-15T11:53:59,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1468679297294027977aec328760b4fe as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe 2024-11-15T11:53:59,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe, entries=200, sequenceid=315, filesize=39.0 K 2024-11-15T11:53:59,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/560c1d62e3dc48d09b452b65ea58ee4c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c 2024-11-15T11:53:59,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T11:53:59,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/e6820e91dd2945d392d42473040bbd7f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f 2024-11-15T11:53:59,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T11:53:59,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1649ms, sequenceid=315, compaction requested=true 2024-11-15T11:53:59,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:59,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:53:59,861 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:59,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:59,861 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:53:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:53:59,862 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:59,862 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:59,862 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:53:59,862 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:53:59,862 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,862 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,862 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=100.3 K 2024-11-15T11:53:59,862 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/30b7087c90164c9191b381a7f0f0326d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.2 K 2024-11-15T11:53:59,862 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,862 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe] 2024-11-15T11:53:59,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:53:59,863 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68fe000702274a03b4930fe5267d9ce0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635350 2024-11-15T11:53:59,863 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b7087c90164c9191b381a7f0f0326d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635350 2024-11-15T11:53:59,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:59,863 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d09f4b47979047148d14d6f8b1fcc5c4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731671635413 2024-11-15T11:53:59,863 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf2640921f040f681457eb1f3a64dba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731671635413 2024-11-15T11:53:59,863 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1468679297294027977aec328760b4fe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637582 2024-11-15T11:53:59,864 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 560c1d62e3dc48d09b452b65ea58ee4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637583 2024-11-15T11:53:59,871 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:59,872 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:59,873 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/0365421f7f6f4de6b55d827922401658 is 50, key is test_row_0/B:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:59,876 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411152e0faa2ca0594b9a8c38b58532cad642_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:59,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T11:53:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,878 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:53:59,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:53:59,879 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411152e0faa2ca0594b9a8c38b58532cad642_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:59,879 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152e0faa2ca0594b9a8c38b58532cad642_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:59,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742417_1593 (size=12609) 2024-11-15T11:53:59,888 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/0365421f7f6f4de6b55d827922401658 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/0365421f7f6f4de6b55d827922401658 2024-11-15T11:53:59,891 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 0365421f7f6f4de6b55d827922401658(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:53:59,891 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:53:59,891 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671639861; duration=0sec 2024-11-15T11:53:59,891 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:53:59,891 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:53:59,891 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:53:59,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:53:59,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:53:59,895 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:53:59,895 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/39c63d6a78ec44ce810b47737caf33bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.2 K 2024-11-15T11:53:59,895 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 39c63d6a78ec44ce810b47737caf33bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731671635350 2024-11-15T11:53:59,896 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c86a84b559d4320ba568232ae03702a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731671635413 2024-11-15T11:53:59,896 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e6820e91dd2945d392d42473040bbd7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637583 2024-11-15T11:53:59,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742418_1594 (size=4469) 2024-11-15T11:53:59,902 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#505 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:59,902 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:53:59,903 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/230d82e12f7d47b683af560c105edabf is 175, key is test_row_0/A:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:59,903 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/9332540b09604b65905566c55a4b6ac6 is 50, key is test_row_0/C:col10/1731671637583/Put/seqid=0 2024-11-15T11:53:59,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c62886a0f52041fb8d19ac6f85af7dab_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671638258/Put/seqid=0 2024-11-15T11:53:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742419_1595 (size=31563) 2024-11-15T11:53:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742420_1596 (size=12609) 2024-11-15T11:53:59,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742421_1597 (size=12454) 2024-11-15T11:53:59,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:53:59,923 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c62886a0f52041fb8d19ac6f85af7dab_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c62886a0f52041fb8d19ac6f85af7dab_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:53:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/24ea1edb41a1429cb915de03b5164c73, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:53:59,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/24ea1edb41a1429cb915de03b5164c73 is 175, key is test_row_0/A:col10/1731671638258/Put/seqid=0 2024-11-15T11:53:59,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742422_1598 (size=31255) 2024-11-15T11:53:59,929 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/24ea1edb41a1429cb915de03b5164c73 2024-11-15T11:53:59,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/e40816c9f7cb4dfc80b72a11758488b9 is 50, key is test_row_0/B:col10/1731671638258/Put/seqid=0 2024-11-15T11:53:59,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742423_1599 (size=12301) 2024-11-15T11:54:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:54:00,316 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/230d82e12f7d47b683af560c105edabf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf 2024-11-15T11:54:00,320 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 230d82e12f7d47b683af560c105edabf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:00,320 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:00,320 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671639861; duration=0sec 2024-11-15T11:54:00,320 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:00,320 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:54:00,324 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/9332540b09604b65905566c55a4b6ac6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/9332540b09604b65905566c55a4b6ac6 2024-11-15T11:54:00,327 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into 9332540b09604b65905566c55a4b6ac6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:00,327 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:00,327 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671639862; duration=0sec 2024-11-15T11:54:00,328 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:00,328 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:54:00,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/e40816c9f7cb4dfc80b72a11758488b9 2024-11-15T11:54:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8f0d0f470e0c4f7ca637d267c93c0723 is 50, key is test_row_0/C:col10/1731671638258/Put/seqid=0 2024-11-15T11:54:00,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742424_1600 (size=12301) 2024-11-15T11:54:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:00,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671700441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,444 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671700441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671700444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671700551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671700551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,746 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8f0d0f470e0c4f7ca637d267c93c0723 2024-11-15T11:54:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/24ea1edb41a1429cb915de03b5164c73 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73 2024-11-15T11:54:00,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73, entries=150, sequenceid=351, filesize=30.5 K 2024-11-15T11:54:00,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/e40816c9f7cb4dfc80b72a11758488b9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9 2024-11-15T11:54:00,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9, entries=150, sequenceid=351, filesize=12.0 K 2024-11-15T11:54:00,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8f0d0f470e0c4f7ca637d267c93c0723 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723 2024-11-15T11:54:00,758 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723, entries=150, sequenceid=351, filesize=12.0 K 2024-11-15T11:54:00,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 698b0d18c06b908fd10b74c4eb30c7d2 in 882ms, sequenceid=351, compaction requested=false 2024-11-15T11:54:00,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:00,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:00,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T11:54:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T11:54:00,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T11:54:00,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7970 sec 2024-11-15T11:54:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:00,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-15T11:54:00,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8010 sec 2024-11-15T11:54:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:00,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:00,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:00,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590492b4910a3437fb2eeba0f8aff225c_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:00,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742425_1601 (size=14994) 2024-11-15T11:54:00,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:00,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111590492b4910a3437fb2eeba0f8aff225c_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590492b4910a3437fb2eeba0f8aff225c_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:00,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68507121d6064c28971edf1dca923e81, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:00,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68507121d6064c28971edf1dca923e81 is 175, key is test_row_0/A:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:00,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742426_1602 (size=39949) 2024-11-15T11:54:00,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671700815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671700816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671700931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:00,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671700931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:00,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T11:54:01,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:01,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:54:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T11:54:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:01,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:54:01,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:54:01,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:54:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671701141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671701141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68507121d6064c28971edf1dca923e81 2024-11-15T11:54:01,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/a60ea1f4998e44f2bd17d3ccca6be6f4 is 50, key is test_row_0/B:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742427_1603 (size=12301) 2024-11-15T11:54:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:01,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T11:54:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T11:54:01,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:01,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:01,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671701450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671701450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T11:54:01,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:01,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:01,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/a60ea1f4998e44f2bd17d3ccca6be6f4 2024-11-15T11:54:01,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8b2f59e96dd34d41bd476d765b5b2cd8 is 50, key is test_row_0/C:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:01,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742428_1604 (size=12301) 2024-11-15T11:54:01,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8b2f59e96dd34d41bd476d765b5b2cd8 2024-11-15T11:54:01,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/68507121d6064c28971edf1dca923e81 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81 2024-11-15T11:54:01,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81, entries=200, sequenceid=366, filesize=39.0 K 2024-11-15T11:54:01,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/a60ea1f4998e44f2bd17d3ccca6be6f4 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4 2024-11-15T11:54:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T11:54:01,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/8b2f59e96dd34d41bd476d765b5b2cd8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8 2024-11-15T11:54:01,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T11:54:01,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 698b0d18c06b908fd10b74c4eb30c7d2 in 855ms, sequenceid=366, compaction requested=true 2024-11-15T11:54:01,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:01,618 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:01,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:54:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:54:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:01,619 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:54:01,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:01,619 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102767 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:01,619 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:01,619 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:54:01,619 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:54:01,619 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,619 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,619 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/0365421f7f6f4de6b55d827922401658, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.3 K 2024-11-15T11:54:01,619 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=100.4 K 2024-11-15T11:54:01,619 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81] 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0365421f7f6f4de6b55d827922401658, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637583 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 230d82e12f7d47b683af560c105edabf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637583 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting e40816c9f7cb4dfc80b72a11758488b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731671638258 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ea1edb41a1429cb915de03b5164c73, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731671638258 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting a60ea1f4998e44f2bd17d3ccca6be6f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640443 2024-11-15T11:54:01,620 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68507121d6064c28971edf1dca923e81, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640439 2024-11-15T11:54:01,625 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:01,625 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/2a87ec6012304b1891ab68f2593f8d1a is 50, key is test_row_0/B:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:01,635 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:01,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742429_1605 (size=12711) 2024-11-15T11:54:01,637 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e80f2a5f0ab74c12976d20728ab9768d_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:01,639 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e80f2a5f0ab74c12976d20728ab9768d_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:01,639 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e80f2a5f0ab74c12976d20728ab9768d_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:01,643 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/2a87ec6012304b1891ab68f2593f8d1a as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2a87ec6012304b1891ab68f2593f8d1a 2024-11-15T11:54:01,647 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 2a87ec6012304b1891ab68f2593f8d1a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:01,647 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:01,647 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671641619; duration=0sec 2024-11-15T11:54:01,647 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:01,647 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:54:01,647 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:01,648 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:01,648 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:54:01,648 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,648 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/9332540b09604b65905566c55a4b6ac6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.3 K 2024-11-15T11:54:01,648 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9332540b09604b65905566c55a4b6ac6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731671637583 2024-11-15T11:54:01,648 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0d0f470e0c4f7ca637d267c93c0723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731671638258 2024-11-15T11:54:01,648 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2f59e96dd34d41bd476d765b5b2cd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640443 2024-11-15T11:54:01,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742430_1606 (size=4469) 2024-11-15T11:54:01,652 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#514 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:01,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/f6798d210fc644a583cfac1ce103823c is 175, key is test_row_0/A:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:01,665 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:01,665 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/09d366c732cb4f0fb1bcc5b8108892ab is 50, key is test_row_0/C:col10/1731671640762/Put/seqid=0 2024-11-15T11:54:01,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742432_1608 (size=12711) 2024-11-15T11:54:01,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742431_1607 (size=31665) 2024-11-15T11:54:01,675 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/09d366c732cb4f0fb1bcc5b8108892ab as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/09d366c732cb4f0fb1bcc5b8108892ab 2024-11-15T11:54:01,682 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into 09d366c732cb4f0fb1bcc5b8108892ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:01,682 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:01,682 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671641619; duration=0sec 2024-11-15T11:54:01,682 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:01,682 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:54:01,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:01,733 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:01,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538c99693f1c0457780af6a717129915b_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671640814/Put/seqid=0 2024-11-15T11:54:01,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742433_1609 (size=12454) 2024-11-15T11:54:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:01,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671701978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671701982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,077 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/f6798d210fc644a583cfac1ce103823c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c 2024-11-15T11:54:02,080 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into f6798d210fc644a583cfac1ce103823c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:02,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:02,080 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671641618; duration=0sec 2024-11-15T11:54:02,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:02,080 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:54:02,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671702090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671702090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:02,147 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111538c99693f1c0457780af6a717129915b_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538c99693f1c0457780af6a717129915b_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/162bd87934bc49aaa5a3a4f26dbeed7b, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/162bd87934bc49aaa5a3a4f26dbeed7b is 175, key is test_row_0/A:col10/1731671640814/Put/seqid=0 2024-11-15T11:54:02,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742434_1610 (size=31255) 2024-11-15T11:54:02,152 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/162bd87934bc49aaa5a3a4f26dbeed7b 2024-11-15T11:54:02,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/1500b5635e7b4b7f9588c002273601cb is 50, key is test_row_0/B:col10/1731671640814/Put/seqid=0 2024-11-15T11:54:02,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742435_1611 (size=12301) 2024-11-15T11:54:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/1500b5635e7b4b7f9588c002273601cb 2024-11-15T11:54:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/6e6cfa015e78478e83223e7227c237a3 is 50, key is test_row_0/C:col10/1731671640814/Put/seqid=0 2024-11-15T11:54:02,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742436_1612 (size=12301) 2024-11-15T11:54:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:02,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671702301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671702301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671702460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,462 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:54:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/6e6cfa015e78478e83223e7227c237a3 2024-11-15T11:54:02,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/162bd87934bc49aaa5a3a4f26dbeed7b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b 2024-11-15T11:54:02,598 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b, entries=150, sequenceid=388, filesize=30.5 K 2024-11-15T11:54:02,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/1500b5635e7b4b7f9588c002273601cb as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb 2024-11-15T11:54:02,601 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb, entries=150, sequenceid=388, filesize=12.0 K 2024-11-15T11:54:02,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/6e6cfa015e78478e83223e7227c237a3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3 2024-11-15T11:54:02,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3, entries=150, sequenceid=388, filesize=12.0 K 2024-11-15T11:54:02,608 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 698b0d18c06b908fd10b74c4eb30c7d2 in 875ms, sequenceid=388, compaction requested=false 2024-11-15T11:54:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T11:54:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T11:54:02,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T11:54:02,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-11-15T11:54:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:02,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:02,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:02,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.4900 sec 2024-11-15T11:54:02,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569d35c217bef4feb8b1b786e2c271609_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:02,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742437_1613 (size=12454) 2024-11-15T11:54:02,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671702649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671702649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671702761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671702761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671702970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671702970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,023 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:03,026 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111569d35c217bef4feb8b1b786e2c271609_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d35c217bef4feb8b1b786e2c271609_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:03,027 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/07f1b4d356c741c897819b24ceba703f, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:03,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/07f1b4d356c741c897819b24ceba703f is 175, key is test_row_0/A:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:03,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742438_1614 (size=31255) 2024-11-15T11:54:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T11:54:03,271 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:03,271 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:54:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T11:54:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:03,272 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:54:03,273 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:54:03,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:54:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671703281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671703281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:03,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:03,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:03,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,431 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/07f1b4d356c741c897819b24ceba703f 2024-11-15T11:54:03,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/fb06aec792d846bf81af62ded94f3c1e is 50, key is test_row_0/B:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:03,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742439_1615 (size=12301) 2024-11-15T11:54:03,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:03,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46232 deadline: 1731671703620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46134 deadline: 1731671703621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,623 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,624 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671703790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671703790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:03,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/fb06aec792d846bf81af62ded94f3c1e 2024-11-15T11:54:03,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/fced41c589484958ad70c4eb2cc460d1 is 50, key is test_row_0/C:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:03,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742440_1616 (size=12301) 2024-11-15T11:54:03,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:03,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:03,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:03,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:04,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:04,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T11:54:04,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/fced41c589484958ad70c4eb2cc460d1 2024-11-15T11:54:04,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/07f1b4d356c741c897819b24ceba703f as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f 2024-11-15T11:54:04,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f, entries=150, sequenceid=402, filesize=30.5 K 2024-11-15T11:54:04,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/fb06aec792d846bf81af62ded94f3c1e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e 2024-11-15T11:54:04,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T11:54:04,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/fced41c589484958ad70c4eb2cc460d1 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1 2024-11-15T11:54:04,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1, entries=150, sequenceid=402, filesize=12.0 K 2024-11-15T11:54:04,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1661ms, sequenceid=402, compaction requested=true 2024-11-15T11:54:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:54:04,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:54:04,273 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:54:04,273 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,273 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,274 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=92.0 K 2024-11-15T11:54:04,274 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2a87ec6012304b1891ab68f2593f8d1a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.4 K 2024-11-15T11:54:04,274 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f] 2024-11-15T11:54:04,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a87ec6012304b1891ab68f2593f8d1a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640443 2024-11-15T11:54:04,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6798d210fc644a583cfac1ce103823c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640443 2024-11-15T11:54:04,274 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1500b5635e7b4b7f9588c002273601cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731671640801 2024-11-15T11:54:04,274 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162bd87934bc49aaa5a3a4f26dbeed7b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731671640801 2024-11-15T11:54:04,275 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fb06aec792d846bf81af62ded94f3c1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:04,275 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f1b4d356c741c897819b24ceba703f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:04,281 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#522 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:04,281 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/76f0a81f7b384f7f9417fa0cadb7e495 is 50, key is test_row_0/B:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:04,283 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:04,291 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111556834224a59046c0afec4a8cdf7a36b4_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:04,293 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111556834224a59046c0afec4a8cdf7a36b4_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:04,293 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111556834224a59046c0afec4a8cdf7a36b4_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742441_1617 (size=12813) 2024-11-15T11:54:04,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742442_1618 (size=4469) 2024-11-15T11:54:04,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,340 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:04,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e10c50fcc4447c9ad6e163cbfd83fc8_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671642642/Put/seqid=0 2024-11-15T11:54:04,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742443_1619 (size=12454) 2024-11-15T11:54:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:04,712 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/76f0a81f7b384f7f9417fa0cadb7e495 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/76f0a81f7b384f7f9417fa0cadb7e495 2024-11-15T11:54:04,715 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 76f0a81f7b384f7f9417fa0cadb7e495(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:04,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:04,715 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671644273; duration=0sec 2024-11-15T11:54:04,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:04,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:54:04,715 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:04,716 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:04,716 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:54:04,716 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:04,716 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/09d366c732cb4f0fb1bcc5b8108892ab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=36.4 K 2024-11-15T11:54:04,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 09d366c732cb4f0fb1bcc5b8108892ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731671640443 2024-11-15T11:54:04,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e6cfa015e78478e83223e7227c237a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731671640801 2024-11-15T11:54:04,717 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting fced41c589484958ad70c4eb2cc460d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:04,722 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:04,723 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/ad41e80299e841be85517289cbb1f34c is 50, key is test_row_0/C:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:04,725 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#523 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:04,725 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/25a2bbd79362499c96154cd7260c776b is 175, key is test_row_0/A:col10/1731671641977/Put/seqid=0 2024-11-15T11:54:04,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742444_1620 (size=12813) 2024-11-15T11:54:04,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742445_1621 (size=31767) 2024-11-15T11:54:04,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:04,752 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156e10c50fcc4447c9ad6e163cbfd83fc8_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e10c50fcc4447c9ad6e163cbfd83fc8_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:04,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/18e99a8fe368431497a35f894b46b460, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:04,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/18e99a8fe368431497a35f894b46b460 is 175, key is test_row_0/A:col10/1731671642642/Put/seqid=0 2024-11-15T11:54:04,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742446_1622 (size=31255) 2024-11-15T11:54:04,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:04,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:04,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:04,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671704825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:04,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671704828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671704941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671704941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:05,130 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/ad41e80299e841be85517289cbb1f34c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ad41e80299e841be85517289cbb1f34c 2024-11-15T11:54:05,134 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/25a2bbd79362499c96154cd7260c776b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b 2024-11-15T11:54:05,136 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into ad41e80299e841be85517289cbb1f34c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:05,136 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:05,136 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671644273; duration=0sec 2024-11-15T11:54:05,136 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:05,136 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:54:05,137 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 25a2bbd79362499c96154cd7260c776b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:05,137 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:05,137 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671644273; duration=0sec 2024-11-15T11:54:05,137 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:05,137 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:54:05,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:05,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671705150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671705150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:05,157 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/18e99a8fe368431497a35f894b46b460 2024-11-15T11:54:05,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/6cfde3e00a3e4234b24c9859056a8492 is 50, key is test_row_0/B:col10/1731671642642/Put/seqid=0 2024-11-15T11:54:05,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742447_1623 (size=12301) 2024-11-15T11:54:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:05,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/6cfde3e00a3e4234b24c9859056a8492 2024-11-15T11:54:05,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/339ef59499e04a0c9b1afe259e6d078c is 50, key is test_row_0/C:col10/1731671642642/Put/seqid=0 2024-11-15T11:54:05,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742448_1624 (size=12301) 2024-11-15T11:54:05,590 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/339ef59499e04a0c9b1afe259e6d078c 2024-11-15T11:54:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/18e99a8fe368431497a35f894b46b460 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460 2024-11-15T11:54:05,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460, entries=150, sequenceid=421, filesize=30.5 K 2024-11-15T11:54:05,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/6cfde3e00a3e4234b24c9859056a8492 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492 2024-11-15T11:54:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,599 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492, entries=150, sequenceid=421, filesize=12.0 K 2024-11-15T11:54:05,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/339ef59499e04a0c9b1afe259e6d078c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c 2024-11-15T11:54:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c, entries=150, sequenceid=421, filesize=12.0 K 2024-11-15T11:54:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1262ms, sequenceid=421, compaction requested=false 2024-11-15T11:54:05,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:05,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:05,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T11:54:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T11:54:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T11:54:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-15T11:54:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3340 sec 2024-11-15T11:54:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:05,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159d73688613784a65bae0f9be1efc41cd_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742449_1625 (size=12454) 2024-11-15T11:54:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159d73688613784a65bae0f9be1efc41cd_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d73688613784a65bae0f9be1efc41cd_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:05,987 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/bd99fb6e292f4bbea1b14f82143e0cbf, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/bd99fb6e292f4bbea1b14f82143e0cbf is 175, key is test_row_0/A:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742450_1626 (size=31251) 2024-11-15T11:54:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:06,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671706022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671706023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671706130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671706130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671706341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671706341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/bd99fb6e292f4bbea1b14f82143e0cbf 2024-11-15T11:54:06,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/931421bde2fe4fb0a72e9861471bb993 is 50, key is test_row_0/B:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742451_1627 (size=9857) 2024-11-15T11:54:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46172 deadline: 1731671706500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,503 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7adf9b3d9d04,44479,1731671471201 for 'test_row_0' in 698b0d18c06b908fd10b74c4eb30c7d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T11:54:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671706650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671706650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:06,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/931421bde2fe4fb0a72e9861471bb993 2024-11-15T11:54:06,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/718beb66ec6644bc978ef3329528f135 is 50, key is test_row_0/C:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:06,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742452_1628 (size=9857) 2024-11-15T11:54:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671707160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671707160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:07,162 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:07,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/718beb66ec6644bc978ef3329528f135 2024-11-15T11:54:07,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/bd99fb6e292f4bbea1b14f82143e0cbf as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf 2024-11-15T11:54:07,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf, entries=150, sequenceid=438, filesize=30.5 K 2024-11-15T11:54:07,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/931421bde2fe4fb0a72e9861471bb993 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993 2024-11-15T11:54:07,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993, entries=100, sequenceid=438, filesize=9.6 K 2024-11-15T11:54:07,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/718beb66ec6644bc978ef3329528f135 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135 2024-11-15T11:54:07,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135, entries=100, sequenceid=438, filesize=9.6 K 2024-11-15T11:54:07,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1248ms, sequenceid=438, compaction requested=true 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:07,222 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:54:07,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:07,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:07,222 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:07,222 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94273 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:54:07,223 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:07,223 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:07,223 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/76f0a81f7b384f7f9417fa0cadb7e495, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=34.2 K 2024-11-15T11:54:07,223 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=92.1 K 2024-11-15T11:54:07,223 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf] 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 76f0a81f7b384f7f9417fa0cadb7e495, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a2bbd79362499c96154cd7260c776b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cfde3e00a3e4234b24c9859056a8492, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731671642642 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e99a8fe368431497a35f894b46b460, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731671642642 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 931421bde2fe4fb0a72e9861471bb993, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671644827 2024-11-15T11:54:07,223 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd99fb6e292f4bbea1b14f82143e0cbf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671644821 2024-11-15T11:54:07,227 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:07,229 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:07,229 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b2417268120a4246a4defa4957ea6a21_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:07,229 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/2adcc293e8834ec78b2093b35f1b3cb3 is 50, key is test_row_0/B:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:07,230 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b2417268120a4246a4defa4957ea6a21_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:07,231 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b2417268120a4246a4defa4957ea6a21_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:07,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742453_1629 (size=12915) 2024-11-15T11:54:07,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742454_1630 (size=4469) 2024-11-15T11:54:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T11:54:07,430 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:07,431 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:54:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T11:54:07,432 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:54:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T11:54:07,433 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:54:07,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:54:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T11:54:07,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:07,585 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:07,634 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#532 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:07,634 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/190298004855431fb4960ae765f4a8f0 is 175, key is test_row_0/A:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:07,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed2fb18d4016418c901c848629aeb081_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671646021/Put/seqid=0 2024-11-15T11:54:07,637 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/2adcc293e8834ec78b2093b35f1b3cb3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2adcc293e8834ec78b2093b35f1b3cb3 2024-11-15T11:54:07,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742455_1631 (size=31976) 2024-11-15T11:54:07,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742456_1632 (size=12454) 2024-11-15T11:54:07,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:07,642 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 2adcc293e8834ec78b2093b35f1b3cb3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:07,642 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:07,642 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=13, startTime=1731671647222; duration=0sec 2024-11-15T11:54:07,642 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:07,642 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:54:07,642 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T11:54:07,643 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T11:54:07,643 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:54:07,643 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:07,643 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ad41e80299e841be85517289cbb1f34c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=34.2 K 2024-11-15T11:54:07,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ed2fb18d4016418c901c848629aeb081_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2fb18d4016418c901c848629aeb081_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:07,644 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting ad41e80299e841be85517289cbb1f34c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731671641977 2024-11-15T11:54:07,644 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 339ef59499e04a0c9b1afe259e6d078c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731671642642 2024-11-15T11:54:07,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3023b8bc509c494c83581d656f698ef6, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:07,644 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 718beb66ec6644bc978ef3329528f135, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671644827 2024-11-15T11:54:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3023b8bc509c494c83581d656f698ef6 is 175, key is test_row_0/A:col10/1731671646021/Put/seqid=0 2024-11-15T11:54:07,648 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/190298004855431fb4960ae765f4a8f0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0 2024-11-15T11:54:07,650 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#534 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:07,650 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/f4d47eec31a94db6a8583af4a17f1cad is 50, key is test_row_0/C:col10/1731671644827/Put/seqid=0 2024-11-15T11:54:07,652 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 190298004855431fb4960ae765f4a8f0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:07,652 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:07,653 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=13, startTime=1731671647222; duration=0sec 2024-11-15T11:54:07,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:07,653 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:54:07,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742457_1633 (size=12915) 2024-11-15T11:54:07,657 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/f4d47eec31a94db6a8583af4a17f1cad as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f4d47eec31a94db6a8583af4a17f1cad 2024-11-15T11:54:07,660 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into f4d47eec31a94db6a8583af4a17f1cad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:07,660 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:07,660 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=13, startTime=1731671647222; duration=0sec 2024-11-15T11:54:07,660 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:07,660 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:54:07,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742458_1634 (size=31255) 2024-11-15T11:54:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T11:54:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T11:54:08,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3023b8bc509c494c83581d656f698ef6 2024-11-15T11:54:08,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 is 50, key is test_row_0/B:col10/1731671646021/Put/seqid=0 2024-11-15T11:54:08,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742459_1635 (size=12301) 2024-11-15T11:54:08,085 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 2024-11-15T11:54:08,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/2731b9e5e7744bb9b521a30be5b0d764 is 50, key is test_row_0/C:col10/1731671646021/Put/seqid=0 2024-11-15T11:54:08,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742460_1636 (size=12301) 2024-11-15T11:54:08,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:08,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671708192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671708192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671708301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671708301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,500 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/2731b9e5e7744bb9b521a30be5b0d764 2024-11-15T11:54:08,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/3023b8bc509c494c83581d656f698ef6 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6 2024-11-15T11:54:08,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6, entries=150, sequenceid=457, filesize=30.5 K 2024-11-15T11:54:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671708511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671708511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 2024-11-15T11:54:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3, entries=150, sequenceid=457, filesize=12.0 K 2024-11-15T11:54:08,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/2731b9e5e7744bb9b521a30be5b0d764 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764 2024-11-15T11:54:08,530 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764, entries=150, sequenceid=457, filesize=12.0 K 2024-11-15T11:54:08,531 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 698b0d18c06b908fd10b74c4eb30c7d2 in 945ms, sequenceid=457, compaction requested=false 2024-11-15T11:54:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:08,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T11:54:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T11:54:08,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T11:54:08,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0990 sec 2024-11-15T11:54:08,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.1030 sec 2024-11-15T11:54:08,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T11:54:08,571 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:08,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T11:54:08,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T11:54:08,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T11:54:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:08,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T11:54:08,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T11:54:08,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:08,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:08,726 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:08,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115324755e8790f49b2afca44c57b3ff17d_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671648190/Put/seqid=0 2024-11-15T11:54:08,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742461_1637 (size=12454) 2024-11-15T11:54:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:08,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. as already flushing 2024-11-15T11:54:08,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671708850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671708852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671708961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671708961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:09,139 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115324755e8790f49b2afca44c57b3ff17d_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115324755e8790f49b2afca44c57b3ff17d_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/381b0b0028934a8d88637605bcac90ab, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:09,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/381b0b0028934a8d88637605bcac90ab is 175, key is test_row_0/A:col10/1731671648190/Put/seqid=0 2024-11-15T11:54:09,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742462_1638 (size=31255) 2024-11-15T11:54:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671709170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671709170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:09,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671709480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671709480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,559 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/381b0b0028934a8d88637605bcac90ab 2024-11-15T11:54:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/9bd8305995d44077a99fe5d7f57cbcc0 is 50, key is test_row_0/B:col10/1731671648190/Put/seqid=0 2024-11-15T11:54:09,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742463_1639 (size=12301) 2024-11-15T11:54:09,573 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T11:54:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:09,969 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/9bd8305995d44077a99fe5d7f57cbcc0 2024-11-15T11:54:09,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 is 50, key is test_row_0/C:col10/1731671648190/Put/seqid=0 2024-11-15T11:54:09,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742464_1640 (size=12301) 2024-11-15T11:54:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46176 deadline: 1731671709990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T11:54:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46202 deadline: 1731671709990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 , the old value is region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T11:54:09,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., hostname=7adf9b3d9d04,44479,1731671471201, seqNum=5 because the exception is null or not the one we care about 2024-11-15T11:54:10,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 2024-11-15T11:54:10,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/381b0b0028934a8d88637605bcac90ab as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab 2024-11-15T11:54:10,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab, entries=150, sequenceid=473, filesize=30.5 K 2024-11-15T11:54:10,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/9bd8305995d44077a99fe5d7f57cbcc0 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0 2024-11-15T11:54:10,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0, entries=150, sequenceid=473, filesize=12.0 K 2024-11-15T11:54:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 2024-11-15T11:54:10,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9, entries=150, sequenceid=473, filesize=12.0 K 2024-11-15T11:54:10,402 INFO [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1676ms, sequenceid=473, compaction requested=true 2024-11-15T11:54:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7adf9b3d9d04:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T11:54:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T11:54:10,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T11:54:10,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-15T11:54:10,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8330 sec 2024-11-15T11:54:10,663 INFO [Thread-2482 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2482. 2024-11-15T11:54:10,663 DEBUG [Thread-2482 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:10,663 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,663 DEBUG [Thread-2482 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:10,663 INFO [Thread-2484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2484. 2024-11-15T11:54:10,664 DEBUG [Thread-2484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:10,664 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:10,664 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,664 INFO [Thread-2478 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2478. 2024-11-15T11:54:10,664 DEBUG [Thread-2478 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:10,664 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,664 DEBUG [Thread-2478 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:10,666 INFO [Thread-2480 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2480. 2024-11-15T11:54:10,666 DEBUG [Thread-2480 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:10,666 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,666 DEBUG [Thread-2480 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:10,666 INFO [Thread-2486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2486. 2024-11-15T11:54:10,666 DEBUG [Thread-2486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:10,666 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,666 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:10,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T11:54:10,721 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44479 {}] regionserver.HRegion(8855): Flush requested on 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:11,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-15T11:54:11,005 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-15T11:54:11,005 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-15T11:54:11,006 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:11,006 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:11,006 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:11,006 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:11,006 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:11,006 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-15T11:54:11,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:11,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:11,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155ffe01c7d6b34c3197fb4fc7a30ddea0_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:11,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742465_1641 (size=12454) 2024-11-15T11:54:11,418 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:11,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155ffe01c7d6b34c3197fb4fc7a30ddea0_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155ffe01c7d6b34c3197fb4fc7a30ddea0_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:11,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1d3e182bc4d24b769bf15d3765e4f40e, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:11,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1d3e182bc4d24b769bf15d3765e4f40e is 175, key is test_row_0/A:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:11,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742466_1642 (size=31255) 2024-11-15T11:54:11,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1d3e182bc4d24b769bf15d3765e4f40e 2024-11-15T11:54:11,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/c5eb6e223ba54e738963974c5cf7c4c9 is 50, key is test_row_0/B:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:11,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742467_1643 (size=12301) 2024-11-15T11:54:11,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/c5eb6e223ba54e738963974c5cf7c4c9 2024-11-15T11:54:11,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/f653d04683c34296ac9fdbc3eb682374 is 50, key is test_row_0/C:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:11,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742468_1644 (size=12301) 2024-11-15T11:54:12,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/f653d04683c34296ac9fdbc3eb682374 2024-11-15T11:54:12,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/1d3e182bc4d24b769bf15d3765e4f40e as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e 2024-11-15T11:54:12,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e, entries=150, sequenceid=494, filesize=30.5 K 2024-11-15T11:54:12,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/c5eb6e223ba54e738963974c5cf7c4c9 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9 2024-11-15T11:54:12,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9, entries=150, sequenceid=494, filesize=12.0 K 2024-11-15T11:54:12,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/f653d04683c34296ac9fdbc3eb682374 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374 2024-11-15T11:54:12,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374, entries=150, sequenceid=494, filesize=12.0 K 2024-11-15T11:54:12,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1263ms, sequenceid=494, compaction requested=true 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T11:54:12,268 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 698b0d18c06b908fd10b74c4eb30c7d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T11:54:12,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:12,268 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:54:12,269 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:54:12,269 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125741 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:54:12,269 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/B is initiating minor compaction (all files) 2024-11-15T11:54:12,269 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/A is initiating minor compaction (all files) 2024-11-15T11:54:12,269 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/B in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:12,269 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/A in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:12,269 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2adcc293e8834ec78b2093b35f1b3cb3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=48.7 K 2024-11-15T11:54:12,269 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=122.8 K 2024-11-15T11:54:12,269 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. files: [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e] 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2adcc293e8834ec78b2093b35f1b3cb3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671642648 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 190298004855431fb4960ae765f4a8f0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671642648 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting b5c4f6ee2d9c45618f8e33bbb38da1d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731671646014 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3023b8bc509c494c83581d656f698ef6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731671646014 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd8305995d44077a99fe5d7f57cbcc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731671648185 2024-11-15T11:54:12,270 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381b0b0028934a8d88637605bcac90ab, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731671648185 2024-11-15T11:54:12,271 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting c5eb6e223ba54e738963974c5cf7c4c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731671648841 2024-11-15T11:54:12,271 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3e182bc4d24b769bf15d3765e4f40e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731671648841 2024-11-15T11:54:12,280 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:12,282 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115122d7862608d4d12bb690c672aadfec9_698b0d18c06b908fd10b74c4eb30c7d2 store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:12,282 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#B#compaction#543 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:12,283 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/7bb5ba3578014f6097f1893bcc83976d is 50, key is test_row_0/B:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:12,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742469_1645 (size=13051) 2024-11-15T11:54:12,300 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115122d7862608d4d12bb690c672aadfec9_698b0d18c06b908fd10b74c4eb30c7d2, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:12,300 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115122d7862608d4d12bb690c672aadfec9_698b0d18c06b908fd10b74c4eb30c7d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:12,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742470_1646 (size=4469) 2024-11-15T11:54:12,731 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#A#compaction#544 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:12,732 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/61ff1dc873dd487cb01cb5c4ba8d9080 is 175, key is test_row_0/A:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:12,734 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/7bb5ba3578014f6097f1893bcc83976d as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7bb5ba3578014f6097f1893bcc83976d 2024-11-15T11:54:12,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742471_1647 (size=32005) 2024-11-15T11:54:12,739 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/B of 698b0d18c06b908fd10b74c4eb30c7d2 into 7bb5ba3578014f6097f1893bcc83976d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:12,739 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:12,739 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/B, priority=12, startTime=1731671652268; duration=0sec 2024-11-15T11:54:12,739 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T11:54:12,739 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:B 2024-11-15T11:54:12,739 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T11:54:12,740 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T11:54:12,740 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1541): 698b0d18c06b908fd10b74c4eb30c7d2/C is initiating minor compaction (all files) 2024-11-15T11:54:12,740 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 698b0d18c06b908fd10b74c4eb30c7d2/C in TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:12,740 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f4d47eec31a94db6a8583af4a17f1cad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374] into tmpdir=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp, totalSize=48.7 K 2024-11-15T11:54:12,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d47eec31a94db6a8583af4a17f1cad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731671642648 2024-11-15T11:54:12,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2731b9e5e7744bb9b521a30be5b0d764, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731671646014 2024-11-15T11:54:12,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf8b2ce5a0e4b28bcd9e080594eb3a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731671648185 2024-11-15T11:54:12,741 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] compactions.Compactor(225): Compacting f653d04683c34296ac9fdbc3eb682374, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731671648841 2024-11-15T11:54:12,751 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 698b0d18c06b908fd10b74c4eb30c7d2#C#compaction#545 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T11:54:12,751 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/c0a6da7bb7fb47b28beae48021ae33f2 is 50, key is test_row_0/C:col10/1731671648841/Put/seqid=0 2024-11-15T11:54:12,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742472_1648 (size=13051) 2024-11-15T11:54:13,148 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/61ff1dc873dd487cb01cb5c4ba8d9080 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/61ff1dc873dd487cb01cb5c4ba8d9080 2024-11-15T11:54:13,152 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/A of 698b0d18c06b908fd10b74c4eb30c7d2 into 61ff1dc873dd487cb01cb5c4ba8d9080(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:13,152 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:13,152 INFO [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/A, priority=12, startTime=1731671652268; duration=0sec 2024-11-15T11:54:13,152 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:13,152 DEBUG [RS:0;7adf9b3d9d04:44479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:A 2024-11-15T11:54:13,158 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/c0a6da7bb7fb47b28beae48021ae33f2 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/c0a6da7bb7fb47b28beae48021ae33f2 2024-11-15T11:54:13,163 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 698b0d18c06b908fd10b74c4eb30c7d2/C of 698b0d18c06b908fd10b74c4eb30c7d2 into c0a6da7bb7fb47b28beae48021ae33f2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T11:54:13,163 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 698b0d18c06b908fd10b74c4eb30c7d2: 2024-11-15T11:54:13,163 INFO [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., storeName=698b0d18c06b908fd10b74c4eb30c7d2/C, priority=12, startTime=1731671652268; duration=0sec 2024-11-15T11:54:13,163 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T11:54:13,163 DEBUG [RS:0;7adf9b3d9d04:44479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 698b0d18c06b908fd10b74c4eb30c7d2:C 2024-11-15T11:54:13,654 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-15T11:54:13,654 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:13,654 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:13,654 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:13,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:13,722 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-15T11:54:13,722 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:13,722 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:13,722 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:13,722 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:16,543 INFO [Thread-2467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2467. 2024-11-15T11:54:16,543 DEBUG [Thread-2467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T11:54:16,543 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:16,544 DEBUG [Thread-2467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:16,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 121 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3139 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3194 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3120 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3120 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3213 2024-11-15T11:54:16,544 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T11:54:16,545 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:54:16,545 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:54:16,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:16,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:16,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:16,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T11:54:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:16,551 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671656551"}]},"ts":"1731671656551"} 2024-11-15T11:54:16,553 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T11:54:16,553 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T11:54:16,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T11:54:16,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, UNASSIGN}] 2024-11-15T11:54:16,557 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, UNASSIGN 2024-11-15T11:54:16,558 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=CLOSING, regionLocation=7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:16,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, UNASSIGN because future has completed 2024-11-15T11:54:16,560 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T11:54:16,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201}] 2024-11-15T11:54:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:16,713 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:16,713 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T11:54:16,714 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 698b0d18c06b908fd10b74c4eb30c7d2, disabling compactions & flushes 2024-11-15T11:54:16,714 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:16,714 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:16,714 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. after waiting 0 ms 2024-11-15T11:54:16,714 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:16,714 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 698b0d18c06b908fd10b74c4eb30c7d2 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=A 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=B 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 698b0d18c06b908fd10b74c4eb30c7d2, store=C 2024-11-15T11:54:16,715 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-15T11:54:16,728 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515563f0622ee448b85c0acda5c0ee92d_698b0d18c06b908fd10b74c4eb30c7d2 is 50, key is test_row_0/A:col10/1731671656541/Put/seqid=0 2024-11-15T11:54:16,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742473_1649 (size=7374) 2024-11-15T11:54:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:17,133 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T11:54:17,141 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111515563f0622ee448b85c0acda5c0ee92d_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515563f0622ee448b85c0acda5c0ee92d_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:17,143 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/58799ebbb9894cb9b0f599eafa81f9b7, store: [table=TestAcidGuarantees family=A region=698b0d18c06b908fd10b74c4eb30c7d2] 2024-11-15T11:54:17,143 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/58799ebbb9894cb9b0f599eafa81f9b7 is 175, key is test_row_0/A:col10/1731671656541/Put/seqid=0 2024-11-15T11:54:17,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742474_1650 (size=13865) 2024-11-15T11:54:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:17,549 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=503, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/58799ebbb9894cb9b0f599eafa81f9b7 2024-11-15T11:54:17,559 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/5f623ede03be48e9beab9dc7096a4a0b is 50, key is test_row_0/B:col10/1731671656541/Put/seqid=0 2024-11-15T11:54:17,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742475_1651 (size=7415) 2024-11-15T11:54:17,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:17,973 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/5f623ede03be48e9beab9dc7096a4a0b 2024-11-15T11:54:17,984 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/ea34ad908fe6473fae1a533abd8dc6a8 is 50, key is test_row_0/C:col10/1731671656541/Put/seqid=0 2024-11-15T11:54:17,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742476_1652 (size=7415) 2024-11-15T11:54:18,389 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/ea34ad908fe6473fae1a533abd8dc6a8 2024-11-15T11:54:18,400 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/A/58799ebbb9894cb9b0f599eafa81f9b7 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/58799ebbb9894cb9b0f599eafa81f9b7 2024-11-15T11:54:18,406 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/58799ebbb9894cb9b0f599eafa81f9b7, entries=50, sequenceid=503, filesize=13.5 K 2024-11-15T11:54:18,407 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/B/5f623ede03be48e9beab9dc7096a4a0b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/5f623ede03be48e9beab9dc7096a4a0b 2024-11-15T11:54:18,411 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/5f623ede03be48e9beab9dc7096a4a0b, entries=50, sequenceid=503, filesize=7.2 K 2024-11-15T11:54:18,412 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/.tmp/C/ea34ad908fe6473fae1a533abd8dc6a8 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ea34ad908fe6473fae1a533abd8dc6a8 2024-11-15T11:54:18,415 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ea34ad908fe6473fae1a533abd8dc6a8, entries=50, sequenceid=503, filesize=7.2 K 2024-11-15T11:54:18,416 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1702ms, sequenceid=503, compaction requested=false 2024-11-15T11:54:18,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e] to archive 2024-11-15T11:54:18,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:54:18,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3554f1671e1a4da2b3e8d7332774524a 2024-11-15T11:54:18,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/8521e4012d024a15be62e9e86dd9672f 2024-11-15T11:54:18,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/5ca9b2c007984e5780636cef3afbd8ea 2024-11-15T11:54:18,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/86a2f09362cf4b37861cf3353b009063 2024-11-15T11:54:18,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/336d81ffb3e94a12bc6a29a488bfc833 2024-11-15T11:54:18,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d2ad5dc5887a4491b9e5692292012c3d 2024-11-15T11:54:18,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68fe000702274a03b4930fe5267d9ce0 2024-11-15T11:54:18,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/d09f4b47979047148d14d6f8b1fcc5c4 2024-11-15T11:54:18,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1468679297294027977aec328760b4fe 2024-11-15T11:54:18,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/230d82e12f7d47b683af560c105edabf 2024-11-15T11:54:18,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/24ea1edb41a1429cb915de03b5164c73 2024-11-15T11:54:18,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/68507121d6064c28971edf1dca923e81 2024-11-15T11:54:18,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/f6798d210fc644a583cfac1ce103823c 2024-11-15T11:54:18,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/162bd87934bc49aaa5a3a4f26dbeed7b 2024-11-15T11:54:18,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/25a2bbd79362499c96154cd7260c776b 2024-11-15T11:54:18,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/07f1b4d356c741c897819b24ceba703f 2024-11-15T11:54:18,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/18e99a8fe368431497a35f894b46b460 2024-11-15T11:54:18,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/190298004855431fb4960ae765f4a8f0 2024-11-15T11:54:18,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/bd99fb6e292f4bbea1b14f82143e0cbf 2024-11-15T11:54:18,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/3023b8bc509c494c83581d656f698ef6 2024-11-15T11:54:18,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/381b0b0028934a8d88637605bcac90ab 2024-11-15T11:54:18,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/1d3e182bc4d24b769bf15d3765e4f40e 2024-11-15T11:54:18,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/18484b6c9a094d82a37d09ec2fa458a5, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/30b7087c90164c9191b381a7f0f0326d, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/0365421f7f6f4de6b55d827922401658, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2a87ec6012304b1891ab68f2593f8d1a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/76f0a81f7b384f7f9417fa0cadb7e495, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2adcc293e8834ec78b2093b35f1b3cb3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9] to archive 2024-11-15T11:54:18,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:54:18,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7ad234c42dee4e759ad0f2394d8e40e6 2024-11-15T11:54:18,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/4b546faf3d714f8db24cdb21549739f1 2024-11-15T11:54:18,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/18484b6c9a094d82a37d09ec2fa458a5 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/18484b6c9a094d82a37d09ec2fa458a5 2024-11-15T11:54:18,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/72dde905da984fbe88be7869f33cb324 2024-11-15T11:54:18,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/81ed03277cf84d0d8fdf57fbb2779832 2024-11-15T11:54:18,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/30b7087c90164c9191b381a7f0f0326d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/30b7087c90164c9191b381a7f0f0326d 2024-11-15T11:54:18,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/8efde97143424f7db3e34b7a72b7a6bf 2024-11-15T11:54:18,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/dbf2640921f040f681457eb1f3a64dba 2024-11-15T11:54:18,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/0365421f7f6f4de6b55d827922401658 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/0365421f7f6f4de6b55d827922401658 2024-11-15T11:54:18,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/560c1d62e3dc48d09b452b65ea58ee4c 2024-11-15T11:54:18,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/e40816c9f7cb4dfc80b72a11758488b9 2024-11-15T11:54:18,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2a87ec6012304b1891ab68f2593f8d1a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2a87ec6012304b1891ab68f2593f8d1a 2024-11-15T11:54:18,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/a60ea1f4998e44f2bd17d3ccca6be6f4 2024-11-15T11:54:18,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/1500b5635e7b4b7f9588c002273601cb 2024-11-15T11:54:18,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/76f0a81f7b384f7f9417fa0cadb7e495 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/76f0a81f7b384f7f9417fa0cadb7e495 2024-11-15T11:54:18,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/fb06aec792d846bf81af62ded94f3c1e 2024-11-15T11:54:18,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/6cfde3e00a3e4234b24c9859056a8492 2024-11-15T11:54:18,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2adcc293e8834ec78b2093b35f1b3cb3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/2adcc293e8834ec78b2093b35f1b3cb3 2024-11-15T11:54:18,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/931421bde2fe4fb0a72e9861471bb993 2024-11-15T11:54:18,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/b5c4f6ee2d9c45618f8e33bbb38da1d3 2024-11-15T11:54:18,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/9bd8305995d44077a99fe5d7f57cbcc0 2024-11-15T11:54:18,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/c5eb6e223ba54e738963974c5cf7c4c9 2024-11-15T11:54:18,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/92f4f9a3fed34b62ade5d1618f3bb5d4, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/39c63d6a78ec44ce810b47737caf33bb, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/9332540b09604b65905566c55a4b6ac6, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/09d366c732cb4f0fb1bcc5b8108892ab, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ad41e80299e841be85517289cbb1f34c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f4d47eec31a94db6a8583af4a17f1cad, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374] to archive 2024-11-15T11:54:18,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T11:54:18,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/818db91827ea4001a06194a5a54234a9 2024-11-15T11:54:18,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/345e5674b5934e9ea30549613a086429 2024-11-15T11:54:18,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/92f4f9a3fed34b62ade5d1618f3bb5d4 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/92f4f9a3fed34b62ade5d1618f3bb5d4 2024-11-15T11:54:18,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/352809db8cfb42c986ae146ce619f265 2024-11-15T11:54:18,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/35484727425b4623927b53df1254ff7a 2024-11-15T11:54:18,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/39c63d6a78ec44ce810b47737caf33bb to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/39c63d6a78ec44ce810b47737caf33bb 2024-11-15T11:54:18,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/568311e47f8d4dae9b22010d2663b02f 2024-11-15T11:54:18,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8c86a84b559d4320ba568232ae03702a 2024-11-15T11:54:18,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/9332540b09604b65905566c55a4b6ac6 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/9332540b09604b65905566c55a4b6ac6 2024-11-15T11:54:18,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/e6820e91dd2945d392d42473040bbd7f 2024-11-15T11:54:18,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8f0d0f470e0c4f7ca637d267c93c0723 2024-11-15T11:54:18,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/09d366c732cb4f0fb1bcc5b8108892ab to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/09d366c732cb4f0fb1bcc5b8108892ab 2024-11-15T11:54:18,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/8b2f59e96dd34d41bd476d765b5b2cd8 2024-11-15T11:54:18,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/6e6cfa015e78478e83223e7227c237a3 2024-11-15T11:54:18,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ad41e80299e841be85517289cbb1f34c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ad41e80299e841be85517289cbb1f34c 2024-11-15T11:54:18,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/fced41c589484958ad70c4eb2cc460d1 2024-11-15T11:54:18,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/339ef59499e04a0c9b1afe259e6d078c 2024-11-15T11:54:18,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f4d47eec31a94db6a8583af4a17f1cad to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f4d47eec31a94db6a8583af4a17f1cad 2024-11-15T11:54:18,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/718beb66ec6644bc978ef3329528f135 2024-11-15T11:54:18,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/2731b9e5e7744bb9b521a30be5b0d764 2024-11-15T11:54:18,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/aaf8b2ce5a0e4b28bcd9e080594eb3a9 2024-11-15T11:54:18,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/f653d04683c34296ac9fdbc3eb682374 2024-11-15T11:54:18,492 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits/506.seqid, newMaxSeqId=506, maxSeqId=4 2024-11-15T11:54:18,493 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. 2024-11-15T11:54:18,493 DEBUG [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 698b0d18c06b908fd10b74c4eb30c7d2: Waiting for close lock at 1731671656713Running coprocessor pre-close hooks at 1731671656714 (+1 ms)Disabling compacts and flushes for region at 1731671656714Disabling writes for close at 1731671656714Obtaining lock to block concurrent updates at 1731671656714Preparing flush snapshotting stores in 698b0d18c06b908fd10b74c4eb30c7d2 at 1731671656714Finished memstore snapshotting TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731671656715 (+1 ms)Flushing stores of TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2. at 1731671656717 (+2 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/A: creating writer at 1731671656717Flushing mob file 698b0d18c06b908fd10b74c4eb30c7d2/A: appending metadata at 1731671656728 (+11 ms)Flushing mob file 698b0d18c06b908fd10b74c4eb30c7d2/A: closing flushed file at 1731671656728Flushing 698b0d18c06b908fd10b74c4eb30c7d2/A: appending metadata at 1731671657143 (+415 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/A: closing flushed file at 1731671657143Flushing 698b0d18c06b908fd10b74c4eb30c7d2/B: creating writer at 1731671657556 (+413 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/B: appending metadata at 1731671657559 (+3 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/B: closing flushed file at 1731671657559Flushing 698b0d18c06b908fd10b74c4eb30c7d2/C: creating writer at 1731671657979 (+420 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/C: appending metadata at 1731671657983 (+4 ms)Flushing 698b0d18c06b908fd10b74c4eb30c7d2/C: closing flushed file at 1731671657983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4daf12b4: reopening flushed file at 1731671658398 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6626c1: reopening flushed file at 1731671658406 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b2e8b02: reopening flushed file at 1731671658411 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 698b0d18c06b908fd10b74c4eb30c7d2 in 1702ms, sequenceid=503, compaction requested=false at 1731671658416 (+5 ms)Writing region close event to WAL at 1731671658489 (+73 ms)Running coprocessor post-close hooks at 1731671658492 (+3 ms)Closed at 1731671658492 2024-11-15T11:54:18,494 INFO [RS_CLOSE_REGION-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,495 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=698b0d18c06b908fd10b74c4eb30c7d2, regionState=CLOSED 2024-11-15T11:54:18,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 because future has completed 2024-11-15T11:54:18,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T11:54:18,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 698b0d18c06b908fd10b74c4eb30c7d2, server=7adf9b3d9d04,44479,1731671471201 in 1.9370 sec 2024-11-15T11:54:18,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-15T11:54:18,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=698b0d18c06b908fd10b74c4eb30c7d2, UNASSIGN in 1.9440 sec 2024-11-15T11:54:18,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T11:54:18,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9490 sec 2024-11-15T11:54:18,507 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731671658506"}]},"ts":"1731671658506"} 2024-11-15T11:54:18,508 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T11:54:18,508 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T11:54:18,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-15T11:54:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T11:54:18,701 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:18,702 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T11:54:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,706 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T11:54:18,708 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,712 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C, FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits] 2024-11-15T11:54:18,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/58799ebbb9894cb9b0f599eafa81f9b7 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/58799ebbb9894cb9b0f599eafa81f9b7 2024-11-15T11:54:18,719 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/61ff1dc873dd487cb01cb5c4ba8d9080 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/A/61ff1dc873dd487cb01cb5c4ba8d9080 2024-11-15T11:54:18,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/5f623ede03be48e9beab9dc7096a4a0b to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/5f623ede03be48e9beab9dc7096a4a0b 2024-11-15T11:54:18,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7bb5ba3578014f6097f1893bcc83976d to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/B/7bb5ba3578014f6097f1893bcc83976d 2024-11-15T11:54:18,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/c0a6da7bb7fb47b28beae48021ae33f2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/c0a6da7bb7fb47b28beae48021ae33f2 2024-11-15T11:54:18,725 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ea34ad908fe6473fae1a533abd8dc6a8 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/C/ea34ad908fe6473fae1a533abd8dc6a8 2024-11-15T11:54:18,727 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits/506.seqid to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2/recovered.edits/506.seqid 2024-11-15T11:54:18,727 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/default/TestAcidGuarantees/698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,727 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T11:54:18,728 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:54:18,728 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T11:54:18,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111509b25baed5cd407786a8f10c23b26b02_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111509b25baed5cd407786a8f10c23b26b02_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515563f0622ee448b85c0acda5c0ee92d_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111515563f0622ee448b85c0acda5c0ee92d_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bea96b908a5419986ad604608a7bb6b_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151bea96b908a5419986ad604608a7bb6b_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,732 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115324755e8790f49b2afca44c57b3ff17d_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115324755e8790f49b2afca44c57b3ff17d_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,733 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538c99693f1c0457780af6a717129915b_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111538c99693f1c0457780af6a717129915b_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115495a65a65fcf46bb917316f6476dd722_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115495a65a65fcf46bb917316f6476dd722_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154bf25312ae65455f963bf36e321f1def_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154bf25312ae65455f963bf36e321f1def_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111551063a1dc5cf474fa5eb49831f510bac_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111551063a1dc5cf474fa5eb49831f510bac_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,736 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155ffe01c7d6b34c3197fb4fc7a30ddea0_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155ffe01c7d6b34c3197fb4fc7a30ddea0_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,737 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d35c217bef4feb8b1b786e2c271609_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111569d35c217bef4feb8b1b786e2c271609_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,737 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e10c50fcc4447c9ad6e163cbfd83fc8_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156e10c50fcc4447c9ad6e163cbfd83fc8_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115787045640b5b45da8750f68ed61a53f0_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115787045640b5b45da8750f68ed61a53f0_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,739 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590492b4910a3437fb2eeba0f8aff225c_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111590492b4910a3437fb2eeba0f8aff225c_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,740 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d73688613784a65bae0f9be1efc41cd_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159d73688613784a65bae0f9be1efc41cd_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,740 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c62886a0f52041fb8d19ac6f85af7dab_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c62886a0f52041fb8d19ac6f85af7dab_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,741 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ce2e260cfac4417987f7e1777b9b9ddb_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ce2e260cfac4417987f7e1777b9b9ddb_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,742 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2fb18d4016418c901c848629aeb081_698b0d18c06b908fd10b74c4eb30c7d2 to hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ed2fb18d4016418c901c848629aeb081_698b0d18c06b908fd10b74c4eb30c7d2 2024-11-15T11:54:18,742 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T11:54:18,744 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,746 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T11:54:18,747 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T11:54:18,748 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,748 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T11:54:18,748 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731671658748"}]},"ts":"9223372036854775807"} 2024-11-15T11:54:18,751 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T11:54:18,751 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 698b0d18c06b908fd10b74c4eb30c7d2, NAME => 'TestAcidGuarantees,,1731671627513.698b0d18c06b908fd10b74c4eb30c7d2.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T11:54:18,751 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T11:54:18,751 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731671658751"}]},"ts":"9223372036854775807"} 2024-11-15T11:54:18,752 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T11:54:18,753 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T11:54:18,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-11-15T11:54:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-15T11:54:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T11:54:18,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T11:54:18,825 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 249), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=10654 (was 10832) 2024-11-15T11:54:18,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T11:54:18,825 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:54:18,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:54:18,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:18,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:18,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T11:54:18,826 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T11:54:18,826 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=387043539, stopped=false 2024-11-15T11:54:18,827 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7adf9b3d9d04,36957,1731671470483 2024-11-15T11:54:18,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T11:54:18,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T11:54:18,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:54:18,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:54:18,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T11:54:18,871 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T11:54:18,871 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:54:18,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T11:54:18,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:18,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T11:54:18,872 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7adf9b3d9d04,44479,1731671471201' ***** 2024-11-15T11:54:18,872 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T11:54:18,872 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T11:54:18,872 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T11:54:18,872 INFO [RS:0;7adf9b3d9d04:44479 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T11:54:18,872 INFO [RS:0;7adf9b3d9d04:44479 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(959): stopping server 7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7adf9b3d9d04:44479. 2024-11-15T11:54:18,873 DEBUG [RS:0;7adf9b3d9d04:44479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T11:54:18,873 DEBUG [RS:0;7adf9b3d9d04:44479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T11:54:18,873 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T11:54:18,874 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T11:54:18,874 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T11:54:18,874 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:18,874 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T11:54:18,874 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T11:54:18,874 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T11:54:18,874 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T11:54:18,874 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T11:54:18,874 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T11:54:18,894 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/info/aa3bc647e4ec4e23a0510665f989df10 is 95, key is TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d./info:/1731671510740/DeleteFamily/seqid=0 2024-11-15T11:54:18,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742477_1653 (size=6514) 2024-11-15T11:54:18,971 INFO [regionserver/7adf9b3d9d04:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T11:54:19,052 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7adf9b3d9d04:36957 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-15T11:54:19,074 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:19,263 INFO [regionserver/7adf9b3d9d04:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T11:54:19,263 INFO [regionserver/7adf9b3d9d04:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T11:54:19,274 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:19,298 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/info/aa3bc647e4ec4e23a0510665f989df10 2024-11-15T11:54:19,315 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/ns/8e9bd63d501a48edbd56bfbf11dad345 is 93, key is TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d./ns:/1731671510695/DeleteFamily/seqid=0 2024-11-15T11:54:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742478_1654 (size=5906) 2024-11-15T11:54:19,475 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:19,675 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:19,718 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/ns/8e9bd63d501a48edbd56bfbf11dad345 2024-11-15T11:54:19,744 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/rep_barrier/d5e6240ac22246bc9c359f60d2ea6b3c is 102, key is TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d./rep_barrier:/1731671510695/DeleteFamily/seqid=0 2024-11-15T11:54:19,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742479_1655 (size=6025) 2024-11-15T11:54:19,875 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:20,075 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:20,149 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/rep_barrier/d5e6240ac22246bc9c359f60d2ea6b3c 2024-11-15T11:54:20,174 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/table/e18a535ee76e4ce18c398693d9bd2f7c is 96, key is TestAcidGuarantees,,1731671475835.18293b2c63865e9766ac3df4bea5a42d./table:/1731671510695/DeleteFamily/seqid=0 2024-11-15T11:54:20,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742480_1656 (size=5936) 2024-11-15T11:54:20,276 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:20,476 DEBUG [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T11:54:20,579 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/table/e18a535ee76e4ce18c398693d9bd2f7c 2024-11-15T11:54:20,583 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/info/aa3bc647e4ec4e23a0510665f989df10 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/info/aa3bc647e4ec4e23a0510665f989df10 2024-11-15T11:54:20,586 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/info/aa3bc647e4ec4e23a0510665f989df10, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T11:54:20,587 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/ns/8e9bd63d501a48edbd56bfbf11dad345 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/ns/8e9bd63d501a48edbd56bfbf11dad345 2024-11-15T11:54:20,589 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/ns/8e9bd63d501a48edbd56bfbf11dad345, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T11:54:20,590 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/rep_barrier/d5e6240ac22246bc9c359f60d2ea6b3c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/rep_barrier/d5e6240ac22246bc9c359f60d2ea6b3c 2024-11-15T11:54:20,593 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/rep_barrier/d5e6240ac22246bc9c359f60d2ea6b3c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T11:54:20,594 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/.tmp/table/e18a535ee76e4ce18c398693d9bd2f7c as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/table/e18a535ee76e4ce18c398693d9bd2f7c 2024-11-15T11:54:20,597 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/table/e18a535ee76e4ce18c398693d9bd2f7c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T11:54:20,598 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false 2024-11-15T11:54:20,603 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T11:54:20,604 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T11:54:20,604 INFO [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T11:54:20,604 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731671658874Running coprocessor pre-close hooks at 1731671658874Disabling compacts and flushes for region at 1731671658874Disabling writes for close at 1731671658874Obtaining lock to block concurrent updates at 1731671658874Preparing flush snapshotting stores in 1588230740 at 1731671658874Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731671658875 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731671658875Flushing 1588230740/info: creating writer at 1731671658875Flushing 1588230740/info: appending metadata at 1731671658894 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731671658894Flushing 1588230740/ns: creating writer at 1731671659301 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731671659315 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731671659315Flushing 1588230740/rep_barrier: creating writer at 1731671659726 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731671659744 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731671659744Flushing 1588230740/table: creating writer at 1731671660157 (+413 ms)Flushing 1588230740/table: appending metadata at 1731671660174 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731671660174Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ba76206: reopening flushed file at 1731671660583 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@219af5ba: reopening flushed file at 1731671660586 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9d59a99: reopening flushed file at 1731671660590 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cbdacaf: reopening flushed file at 1731671660593 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1724ms, sequenceid=90, compaction requested=false at 1731671660598 (+5 ms)Writing region close event to WAL at 1731671660600 (+2 ms)Running coprocessor post-close hooks at 1731671660604 (+4 ms)Closed at 1731671660604 2024-11-15T11:54:20,604 DEBUG [RS_CLOSE_META-regionserver/7adf9b3d9d04:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T11:54:20,676 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(976): stopping server 7adf9b3d9d04,44479,1731671471201; all regions closed. 2024-11-15T11:54:20,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741834_1010 (size=25344) 2024-11-15T11:54:20,687 DEBUG [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/oldWALs 2024-11-15T11:54:20,687 INFO [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7adf9b3d9d04%2C44479%2C1731671471201.meta:.meta(num 1731671475336) 2024-11-15T11:54:20,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741832_1008 (size=20718973) 2024-11-15T11:54:20,692 DEBUG [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/oldWALs 2024-11-15T11:54:20,692 INFO [RS:0;7adf9b3d9d04:44479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7adf9b3d9d04%2C44479%2C1731671471201:(num 1731671474390) 2024-11-15T11:54:20,692 DEBUG [RS:0;7adf9b3d9d04:44479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T11:54:20,692 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T11:54:20,692 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T11:54:20,693 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.ChoreService(370): Chore service for: regionserver/7adf9b3d9d04:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T11:54:20,693 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T11:54:20,693 INFO [regionserver/7adf9b3d9d04:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T11:54:20,694 INFO [RS:0;7adf9b3d9d04:44479 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44479 2024-11-15T11:54:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7adf9b3d9d04,44479,1731671471201 2024-11-15T11:54:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T11:54:20,745 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T11:54:20,746 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7adf9b3d9d04,44479,1731671471201] 2024-11-15T11:54:20,761 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7adf9b3d9d04,44479,1731671471201 already deleted, retry=false 2024-11-15T11:54:20,762 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7adf9b3d9d04,44479,1731671471201 expired; onlineServers=0 2024-11-15T11:54:20,762 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7adf9b3d9d04,36957,1731671470483' ***** 2024-11-15T11:54:20,762 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T11:54:20,762 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T11:54:20,762 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T11:54:20,762 DEBUG [M:0;7adf9b3d9d04:36957 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T11:54:20,762 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T11:54:20,762 DEBUG [M:0;7adf9b3d9d04:36957 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T11:54:20,762 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.large.0-1731671474056 {}] cleaner.HFileCleaner(306): Exit Thread[master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.large.0-1731671474056,5,FailOnTimeoutGroup] 2024-11-15T11:54:20,762 DEBUG [master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.small.0-1731671474057 {}] cleaner.HFileCleaner(306): Exit Thread[master/7adf9b3d9d04:0:becomeActiveMaster-HFileCleaner.small.0-1731671474057,5,FailOnTimeoutGroup] 2024-11-15T11:54:20,763 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.ChoreService(370): Chore service for: master/7adf9b3d9d04:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T11:54:20,763 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T11:54:20,763 DEBUG [M:0;7adf9b3d9d04:36957 {}] master.HMaster(1795): Stopping service threads 2024-11-15T11:54:20,763 INFO [M:0;7adf9b3d9d04:36957 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T11:54:20,763 INFO [M:0;7adf9b3d9d04:36957 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T11:54:20,764 ERROR [M:0;7adf9b3d9d04:36957 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T11:54:20,764 INFO [M:0;7adf9b3d9d04:36957 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T11:54:20,765 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T11:54:20,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T11:54:20,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T11:54:20,770 DEBUG [M:0;7adf9b3d9d04:36957 {}] zookeeper.ZKUtil(347): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T11:54:20,770 WARN [M:0;7adf9b3d9d04:36957 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T11:54:20,771 INFO [M:0;7adf9b3d9d04:36957 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/.lastflushedseqids 2024-11-15T11:54:20,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742481_1657 (size=72) 2024-11-15T11:54:20,837 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T11:54:20,838 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T11:54:20,838 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T11:54:20,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T11:54:20,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44479-0x1013f9f41130001, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T11:54:20,854 INFO [RS:0;7adf9b3d9d04:44479 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T11:54:20,854 INFO [RS:0;7adf9b3d9d04:44479 {}] regionserver.HRegionServer(1031): Exiting; stopping=7adf9b3d9d04,44479,1731671471201; zookeeper connection closed. 2024-11-15T11:54:20,855 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55733c38 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55733c38 2024-11-15T11:54:20,855 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T11:54:21,185 INFO [M:0;7adf9b3d9d04:36957 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T11:54:21,186 INFO [M:0;7adf9b3d9d04:36957 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T11:54:21,186 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T11:54:21,187 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:54:21,187 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:54:21,187 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T11:54:21,187 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:54:21,187 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=774.48 KB heapSize=949.02 KB 2024-11-15T11:54:21,208 DEBUG [M:0;7adf9b3d9d04:36957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccbc365b87be4568918d317cb662b293 is 82, key is hbase:meta,,1/info:regioninfo/1731671475422/Put/seqid=0 2024-11-15T11:54:21,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742482_1658 (size=5672) 2024-11-15T11:54:21,612 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccbc365b87be4568918d317cb662b293 2024-11-15T11:54:21,633 DEBUG [M:0;7adf9b3d9d04:36957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92254625601e40cebacaa89c24fae116 is 2479, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1731671513943/Put/seqid=0 2024-11-15T11:54:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742483_1659 (size=45350) 2024-11-15T11:54:22,037 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=773.93 KB at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92254625601e40cebacaa89c24fae116 2024-11-15T11:54:22,041 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92254625601e40cebacaa89c24fae116 2024-11-15T11:54:22,057 DEBUG [M:0;7adf9b3d9d04:36957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b10cdbe44d994508939b358a5897688b is 69, key is 7adf9b3d9d04,44479,1731671471201/rs:state/1731671474119/Put/seqid=0 2024-11-15T11:54:22,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073742484_1660 (size=5156) 2024-11-15T11:54:22,461 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b10cdbe44d994508939b358a5897688b 2024-11-15T11:54:22,470 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ccbc365b87be4568918d317cb662b293 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ccbc365b87be4568918d317cb662b293 2024-11-15T11:54:22,474 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ccbc365b87be4568918d317cb662b293, entries=8, sequenceid=2154, filesize=5.5 K 2024-11-15T11:54:22,475 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/92254625601e40cebacaa89c24fae116 as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/92254625601e40cebacaa89c24fae116 2024-11-15T11:54:22,479 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92254625601e40cebacaa89c24fae116 2024-11-15T11:54:22,479 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/92254625601e40cebacaa89c24fae116, entries=170, sequenceid=2154, filesize=44.3 K 2024-11-15T11:54:22,479 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b10cdbe44d994508939b358a5897688b as hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b10cdbe44d994508939b358a5897688b 2024-11-15T11:54:22,483 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46663/user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b10cdbe44d994508939b358a5897688b, entries=1, sequenceid=2154, filesize=5.0 K 2024-11-15T11:54:22,483 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(3140): Finished flush of dataSize ~774.48 KB/793072, heapSize ~948.73 KB/971496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2154, compaction requested=false 2024-11-15T11:54:22,484 INFO [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T11:54:22,485 DEBUG [M:0;7adf9b3d9d04:36957 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731671661186Disabling compacts and flushes for region at 1731671661186Disabling writes for close at 1731671661187 (+1 ms)Obtaining lock to block concurrent updates at 1731671661187Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731671661187Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793072, getHeapSize=971736, getOffHeapSize=0, getCellsCount=2349 at 1731671661188 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731671661189 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731671661189Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731671661208 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731671661208Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731671661615 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731671661633 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731671661633Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731671662041 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731671662057 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731671662057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bdcd40a: reopening flushed file at 1731671662469 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8eb5b9f: reopening flushed file at 1731671662475 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c9679ea: reopening flushed file at 1731671662479 (+4 ms)Finished flush of dataSize ~774.48 KB/793072, heapSize ~948.73 KB/971496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2154, compaction requested=false at 1731671662483 (+4 ms)Writing region close event to WAL at 1731671662484 (+1 ms)Closed at 1731671662484 2024-11-15T11:54:22,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42045 is added to blk_1073741830_1006 (size=934281) 2024-11-15T11:54:22,487 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ea21fdce-c80b-c882-cf86-9cc1e929f186/MasterData/WALs/7adf9b3d9d04,36957,1731671470483/7adf9b3d9d04%2C36957%2C1731671470483.1731671473395 not finished, retry = 0 2024-11-15T11:54:22,588 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T11:54:22,589 INFO [M:0;7adf9b3d9d04:36957 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T11:54:22,589 INFO [M:0;7adf9b3d9d04:36957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36957 2024-11-15T11:54:22,589 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T11:54:22,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T11:54:22,745 INFO [M:0;7adf9b3d9d04:36957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T11:54:22,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36957-0x1013f9f41130000, quorum=127.0.0.1:54275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T11:54:22,754 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T11:54:22,758 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T11:54:22,758 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T11:54:22,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T11:54:22,758 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.log.dir/,STOPPED} 2024-11-15T11:54:22,761 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T11:54:22,761 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T11:54:22,761 WARN [BP-588673048-172.17.0.2-1731671467135 heartbeating to localhost/127.0.0.1:46663 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T11:54:22,761 WARN [BP-588673048-172.17.0.2-1731671467135 heartbeating to localhost/127.0.0.1:46663 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-588673048-172.17.0.2-1731671467135 (Datanode Uuid efe92078-cba5-4929-bf50-7576633a7deb) service to localhost/127.0.0.1:46663 2024-11-15T11:54:22,764 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data1/current/BP-588673048-172.17.0.2-1731671467135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T11:54:22,765 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/cluster_23fcf7f0-86fa-90f9-b286-68ec0bcc3ec0/data/data2/current/BP-588673048-172.17.0.2-1731671467135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T11:54:22,765 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T11:54:22,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T11:54:22,774 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T11:54:22,774 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T11:54:22,774 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T11:54:22,774 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1ff44774-a76f-9499-2688-dc32da4e8646/hadoop.log.dir/,STOPPED} 2024-11-15T11:54:22,792 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T11:54:22,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down